00000001:02000400:2.0F:1788844868.701695:0:68166:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 210: Local NI recovery checks ======== 01:21:07 (1788844867) 00000400:00000010:3.0F:1788844869.201740:0:68199:0:(lib-cpt.c:67:cfs_cpt_table_alloc()) alloc '(cptab)': 56 at 00000000eb0b11e7. 00000400:00000010:3.0:1788844869.201768:0:68199:0:(lib-cpt.c:76:cfs_cpt_table_alloc()) alloc '(cptab->ctb_nodemask)': 128 at 0000000093d9ed71. 00000400:00000010:3.0:1788844869.201771:0:68199:0:(lib-cpt.c:80:cfs_cpt_table_alloc()) alloc '(cptab->ctb_cpu2cpt)': 16 at 00000000f14711f1. 00000400:00000010:3.0:1788844869.201773:0:68199:0:(lib-cpt.c:87:cfs_cpt_table_alloc()) alloc '(cptab->ctb_node2cpt)': 4 at 00000000439298d6. 00000400:00000010:3.0:1788844869.201775:0:68199:0:(lib-cpt.c:94:cfs_cpt_table_alloc()) alloc '(cptab->ctb_parts)': 32 at 000000003daedcdc. 00000400:00000010:3.0:1788844869.201794:0:68199:0:(lib-cpt.c:106:cfs_cpt_table_alloc()) alloc '(part->cpt_nodemask)': 128 at 00000000fd1a90a8. 00000400:00000010:3.0:1788844869.201797:0:68199:0:(lib-cpt.c:110:cfs_cpt_table_alloc()) alloc '(part->cpt_distance)': 4 at 00000000b230c0a3. 00000400:00000010:0.0F:1788844869.998899:0:68199:0:(lock.c:52:cfs_percpt_lock_create()) alloc '(pcl)': 24 at 00000000a799a61b. 00000400:00000010:0.0:1788844869.998928:0:68199:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000004bf7388d. 00000400:00000010:0.0:1788844869.998935:0:68199:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ec4b4fce. 00000400:00000010:0.0:1788844869.998938:0:68199:0:(lock.c:52:cfs_percpt_lock_create()) alloc '(pcl)': 24 at 00000000a91e8de2. 00000400:00000010:0.0:1788844869.998939:0:68199:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000081a5f59d. 00000400:00000010:0.0:1788844869.998941:0:68199:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e6a5a9bd. 00000400:00000200:0.0:1788844869.999135:0:68199:0:(api-ni.c:978:lnet_register_lnd()) lo LND registered 00000400:00000200:2.0:1788844870.185678:0:68215:0:(api-ni.c:978:lnet_register_lnd()) tcp LND registered 00000001:02000400:2.0:1788844870.212249:0:68217:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure 00000400:00000010:2.0:1788844870.224874:0:68218:0:(api-ni.c:676:lnet_create_remote_nets_table()) alloc '(hash)': 1024 at 000000007b1a5f43. 00000400:00000010:2.0:1788844870.224887:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d72a03a9. 00000400:00000010:2.0:1788844870.224892:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000cadf9311. 00000400:00000010:2.0:1788844870.224894:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000381127ab. 00000400:00000010:2.0:1788844870.224896:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 000000008c32b593. 00000400:00000010:2.0:1788844870.224903:0:68218:0:(peer.c:109:lnet_peer_tables_create()) alloc '(hash)': 8192 at 000000001c3d3225. 00000400:00000010:2.0:1788844870.224908:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e275d772. 00000400:00000010:2.0:1788844870.224910:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000e85a91b5. 00000400:00000010:2.0:1788844870.224914:0:68218:0:(lib-msg.c:1338:lnet_msg_container_setup()) alloc '(container->msc_finalizers)': 32 at 0000000020509cf5. 00000400:00000010:2.0:1788844870.224918:0:68218:0:(lib-msg.c:1348:lnet_msg_container_setup()) alloc '(container->msc_resenders)': 32 at 00000000231f6131. 00000400:00000010:2.0:1788844870.224992:0:68218:0:(api-ni.c:1227:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 00000000b244745d. 00000400:00000010:2.0:1788844870.225011:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000f3c35bf7. 00000400:00000010:2.0:1788844870.225013:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000be36d944. 00000400:00000010:2.0:1788844870.225071:0:68218:0:(api-ni.c:1227:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 0000000006e052d1. 00000400:00000010:2.0:1788844870.225089:0:68218:0:(lib-ptl.c:864:lnet_portals_create()) alloc '(the_lnet.ln_portals)': 512 at 00000000fa760abd. 00000400:00000010:2.0:1788844870.225092:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000009822bda0. 00000400:00000010:2.0:1788844870.225095:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000000ba5a3e3. 00000400:00000010:2.0:1788844870.225097:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000aacf4d98. 00000400:00000010:2.0:1788844870.225103:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000078e45bfe. 00000400:00000010:2.0:1788844870.225106:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000008d92d1d0. 00000400:00000010:2.0:1788844870.225108:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000022882eba. 00000400:00000010:2.0:1788844870.225110:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006eef58f3. 00000400:00000010:2.0:1788844870.225116:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000073a778bd. 00000400:00000010:2.0:1788844870.225118:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000509589b9. 00000400:00000010:2.0:1788844870.225120:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e622a591. 00000400:00000010:2.0:1788844870.225122:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004a12cee2. 00000400:00000010:2.0:1788844870.225127:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008608d7f3. 00000400:00000010:2.0:1788844870.225130:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000033331e74. 00000400:00000010:2.0:1788844870.225132:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001fb966d6. 00000400:00000010:2.0:1788844870.225139:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a985dd30. 00000400:00000010:2.0:1788844870.225144:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000046f7e0d3. 00000400:00000010:2.0:1788844870.225146:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000006a673634. 00000400:00000010:2.0:1788844870.225148:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ac38bab1. 00000400:00000010:2.0:1788844870.225150:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000af60611f. 00000400:00000010:2.0:1788844870.225153:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e54e880c. 00000400:00000010:2.0:1788844870.225156:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000040e59d94. 00000400:00000010:2.0:1788844870.225157:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000003dcaf6c3. 00000400:00000010:2.0:1788844870.225159:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004c11046f. 00000400:00000010:2.0:1788844870.225163:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004e1cda6e. 00000400:00000010:2.0:1788844870.225165:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000082aaed87. 00000400:00000010:2.0:1788844870.225167:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000083817372. 00000400:00000010:2.0:1788844870.225169:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000794d30fb. 00000400:00000010:2.0:1788844870.225173:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e0fb5a6a. 00000400:00000010:2.0:1788844870.225176:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000006ac195db. 00000400:00000010:2.0:1788844870.225177:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e96c2101. 00000400:00000010:2.0:1788844870.225179:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008aed253d. 00000400:00000010:2.0:1788844870.225184:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006051c759. 00000400:00000010:2.0:1788844870.225186:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000be812680. 00000400:00000010:2.0:1788844870.225188:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000673ec8c3. 00000400:00000010:2.0:1788844870.225190:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000487f3d81. 00000400:00000010:2.0:1788844870.225194:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003c7946d5. 00000400:00000010:2.0:1788844870.225196:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000be64e6df. 00000400:00000010:2.0:1788844870.225198:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000006f91e6c2. 00000400:00000010:2.0:1788844870.225200:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000746ba041. 00000400:00000010:2.0:1788844870.225203:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000049f837f7. 00000400:00000010:2.0:1788844870.225206:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000533aebf8. 00000400:00000010:2.0:1788844870.225208:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002de19088. 00000400:00000010:2.0:1788844870.225210:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d2188600. 00000400:00000010:2.0:1788844870.225214:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004c570512. 00000400:00000010:2.0:1788844870.225216:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000fbcb27d0. 00000400:00000010:2.0:1788844870.225218:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000046f2779d. 00000400:00000010:2.0:1788844870.225226:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004c7598be. 00000400:00000010:2.0:1788844870.225232:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000beeb68e0. 00000400:00000010:2.0:1788844870.225235:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000636096a3. 00000400:00000010:2.0:1788844870.225236:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000b3ba702c. 00000400:00000010:2.0:1788844870.225237:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f858c11c. 00000400:00000010:2.0:1788844870.225241:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000048876783. 00000400:00000010:2.0:1788844870.225243:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000003f0b1dcc. 00000400:00000010:2.0:1788844870.225245:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000cb33196d. 00000400:00000010:2.0:1788844870.225246:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f42e59ce. 00000400:00000010:2.0:1788844870.225250:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000aa6f72a4. 00000400:00000010:2.0:1788844870.225252:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000be60938d. 00000400:00000010:2.0:1788844870.225253:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ed74736f. 00000400:00000010:2.0:1788844870.225255:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e5bf5a26. 00000400:00000010:2.0:1788844870.225258:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000071fab5be. 00000400:00000010:2.0:1788844870.225260:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000007712bc2f. 00000400:00000010:2.0:1788844870.225262:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000092d5e100. 00000400:00000010:2.0:1788844870.225264:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004f6fe3dd. 00000400:00000010:2.0:1788844870.225268:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000052a76679. 00000400:00000010:2.0:1788844870.225270:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000410237bb. 00000400:00000010:2.0:1788844870.225272:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000003383df86. 00000400:00000010:2.0:1788844870.225273:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000bde7bc46. 00000400:00000010:2.0:1788844870.225277:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002383e933. 00000400:00000010:2.0:1788844870.225279:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000f05f8179. 00000400:00000010:2.0:1788844870.225280:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000003e4ffdfb. 00000400:00000010:2.0:1788844870.225282:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fca55343. 00000400:00000010:2.0:1788844870.225286:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004c3f562e. 00000400:00000010:2.0:1788844870.225288:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000003e97ebac. 00000400:00000010:2.0:1788844870.225290:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000008325b807. 00000400:00000010:2.0:1788844870.225291:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000071003de9. 00000400:00000010:2.0:1788844870.225295:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f8fd273d. 00000400:00000010:2.0:1788844870.225298:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000d4e6dfac. 00000400:00000010:2.0:1788844870.225300:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000c5c66057. 00000400:00000010:2.0:1788844870.225306:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c8960bc1. 00000400:00000010:2.0:1788844870.225310:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000732bddc8. 00000400:00000010:2.0:1788844870.225313:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e6f3a77a. 00000400:00000010:2.0:1788844870.225314:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001c0f80ec. 00000400:00000010:2.0:1788844870.225316:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a8e6c637. 00000400:00000010:2.0:1788844870.225319:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000000b186f96. 00000400:00000010:2.0:1788844870.225322:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000000d8ff21a. 00000400:00000010:2.0:1788844870.225324:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000003307edf7. 00000400:00000010:2.0:1788844870.225325:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000086157974. 00000400:00000010:2.0:1788844870.225329:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000fcd20a28. 00000400:00000010:2.0:1788844870.225332:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000060c3a218. 00000400:00000010:2.0:1788844870.225334:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000003c416875. 00000400:00000010:2.0:1788844870.225336:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c6360905. 00000400:00000010:2.0:1788844870.225339:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f830d99c. 00000400:00000010:2.0:1788844870.225342:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000002b4ef567. 00000400:00000010:2.0:1788844870.225343:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000051b31e9e. 00000400:00000010:2.0:1788844870.225345:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000049eb34aa. 00000400:00000010:2.0:1788844870.225349:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000058d5f192. 00000400:00000010:2.0:1788844870.225352:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000c29af030. 00000400:00000010:2.0:1788844870.225354:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d53e8531. 00000400:00000010:2.0:1788844870.225355:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008e3f0f17. 00000400:00000010:2.0:1788844870.225360:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004b3231fa. 00000400:00000010:2.0:1788844870.225362:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000000e3ed993. 00000400:00000010:2.0:1788844870.225364:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000c3ca6ac0. 00000400:00000010:2.0:1788844870.225365:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000dfd23c39. 00000400:00000010:2.0:1788844870.225369:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000c587f8d0. 00000400:00000010:2.0:1788844870.225371:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e4078625. 00000400:00000010:2.0:1788844870.225373:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000fe98c810. 00000400:00000010:2.0:1788844870.225374:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f8e8bdff. 00000400:00000010:2.0:1788844870.225379:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000035c7b992. 00000400:00000010:2.0:1788844870.225381:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000001c2ac2af. 00000400:00000010:2.0:1788844870.225383:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000082722c85. 00000400:00000010:2.0:1788844870.225390:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000066542ff8. 00000400:00000010:2.0:1788844870.225394:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005aefee44. 00000400:00000010:2.0:1788844870.225397:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000bad21083. 00000400:00000010:2.0:1788844870.225399:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ad2ebd98. 00000400:00000010:2.0:1788844870.225401:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000082b9c69. 00000400:00000010:2.0:1788844870.225405:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000012bb5fad. 00000400:00000010:2.0:1788844870.225407:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000039a56ea3. 00000400:00000010:2.0:1788844870.225409:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000dc923f14. 00000400:00000010:2.0:1788844870.225411:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000207bc300. 00000400:00000010:2.0:1788844870.225414:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b7b45c03. 00000400:00000010:2.0:1788844870.225416:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000bf7e8b72. 00000400:00000010:2.0:1788844870.225417:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000550e30e3. 00000400:00000010:2.0:1788844870.225419:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fe5276fb. 00000400:00000010:2.0:1788844870.225431:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000025943f65. 00000400:00000010:2.0:1788844870.225435:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000006afbcb62. 00000400:00000010:2.0:1788844870.225437:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d8c62805. 00000400:00000010:2.0:1788844870.225438:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ba5dbec0. 00000400:00000010:2.0:1788844870.225442:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000007e760ea. 00000400:00000010:2.0:1788844870.225445:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000015374619. 00000400:00000010:2.0:1788844870.225447:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001e0fde64. 00000400:00000010:2.0:1788844870.225448:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d4109edb. 00000400:00000010:2.0:1788844870.225452:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000535203ad. 00000400:00000010:2.0:1788844870.225455:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ca425fd7. 00000400:00000010:2.0:1788844870.225457:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000008ce46f14. 00000400:00000010:2.0:1788844870.225458:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e89178ad. 00000400:00000010:2.0:1788844870.225462:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000099676174. 00000400:00000010:2.0:1788844870.225464:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000d2b64378. 00000400:00000010:2.0:1788844870.225466:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a6b8db6d. 00000400:00000010:2.0:1788844870.225468:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002d5f1d21. 00000400:00000010:2.0:1788844870.225476:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005d51b27f. 00000400:00000010:2.0:1788844870.225480:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000f5d7249b. 00000400:00000010:2.0:1788844870.225481:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d19d5c46. 00000400:00000010:2.0:1788844870.225486:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008ddde762. 00000400:00000010:2.0:1788844870.225491:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000148b82bf. 00000400:00000010:2.0:1788844870.225493:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000002e8d245c. 00000400:00000010:2.0:1788844870.225494:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000007c655c6c. 00000400:00000010:2.0:1788844870.225496:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000afcd23c6. 00000400:00000010:2.0:1788844870.225499:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ce23c6c8. 00000400:00000010:2.0:1788844870.225501:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000584e00a3. 00000400:00000010:2.0:1788844870.225504:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000694f14af. 00000400:00000010:2.0:1788844870.225505:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008c904632. 00000400:00000010:2.0:1788844870.225508:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008f2d5864. 00000400:00000010:2.0:1788844870.225511:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000000c9f347a. 00000400:00000010:2.0:1788844870.225512:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000032772fda. 00000400:00000010:2.0:1788844870.225514:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002b278a64. 00000400:00000010:2.0:1788844870.225524:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e492d794. 00000400:00000010:2.0:1788844870.225527:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000bba856b4. 00000400:00000010:2.0:1788844870.225528:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000003752fe09. 00000400:00000010:2.0:1788844870.225530:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003d4a08f6. 00000400:00000010:2.0:1788844870.225535:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ea9620cd. 00000400:00000010:2.0:1788844870.225537:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000008ce8d092. 00000400:00000010:2.0:1788844870.225539:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000095e0fb09. 00000400:00000010:2.0:1788844870.225541:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000043d7f657. 00000400:00000010:2.0:1788844870.225545:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f4841a6c. 00000400:00000010:2.0:1788844870.225548:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000004df8bfca. 00000400:00000010:2.0:1788844870.225549:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a2513ec0. 00000400:00000010:2.0:1788844870.225551:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001ba0346c. 00000400:00000010:2.0:1788844870.225556:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e29cd06f. 00000400:00000010:2.0:1788844870.225558:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000000d0584b1. 00000400:00000010:2.0:1788844870.225559:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000814a01ff. 00000400:00000010:2.0:1788844870.225561:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fbb73f0c. 00000400:00000010:2.0:1788844870.225569:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000094e9313b. 00000400:00000010:2.0:1788844870.225573:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000090c9d724. 00000400:00000010:2.0:1788844870.225574:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000000868a3d0. 00000400:00000010:2.0:1788844870.225578:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001a09bc38. 00000400:00000010:2.0:1788844870.225583:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000090d28e3a. 00000400:00000010:2.0:1788844870.225585:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000439803c5. 00000400:00000010:2.0:1788844870.225587:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000006ede8a16. 00000400:00000010:2.0:1788844870.225588:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003c128bf3. 00000400:00000010:2.0:1788844870.225593:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b6393d4e. 00000400:00000010:2.0:1788844870.225595:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000f62836a0. 00000400:00000010:2.0:1788844870.225597:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d61be514. 00000400:00000010:2.0:1788844870.225598:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b8c5ba7f. 00000400:00000010:2.0:1788844870.225602:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004d403923. 00000400:00000010:2.0:1788844870.225605:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a66f45f9. 00000400:00000010:2.0:1788844870.225606:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d4c04789. 00000400:00000010:2.0:1788844870.225608:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000074d2f83e. 00000400:00000010:2.0:1788844870.225617:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000cce5a98d. 00000400:00000010:2.0:1788844870.225620:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000aa548f2f. 00000400:00000010:2.0:1788844870.225622:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000000da25629. 00000400:00000010:2.0:1788844870.225624:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002accb2c7. 00000400:00000010:2.0:1788844870.225627:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b06228b2. 00000400:00000010:2.0:1788844870.225629:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000d34354f1. 00000400:00000010:2.0:1788844870.225631:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000fe9f290d. 00000400:00000010:2.0:1788844870.225632:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004a544abe. 00000400:00000010:2.0:1788844870.225635:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000011c2bc27. 00000400:00000010:2.0:1788844870.225638:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ac62a264. 00000400:00000010:2.0:1788844870.225640:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000004d22b8e9. 00000400:00000010:2.0:1788844870.225642:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ad341a92. 00000400:00000010:2.0:1788844870.225645:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001c0ab122. 00000400:00000010:2.0:1788844870.225648:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000006e877309. 00000400:00000010:2.0:1788844870.225658:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d6e54e6a. 00000400:00000010:2.0:1788844870.225660:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000024ef0b92. 00000400:00000010:2.0:1788844870.225668:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000041fda1eb. 00000400:00000010:2.0:1788844870.225671:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e4cb4805. 00000400:00000010:2.0:1788844870.225673:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005e19d104. 00000400:00000010:2.0:1788844870.225677:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001e8ecbb5. 00000400:00000010:2.0:1788844870.225681:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000091cba7af. 00000400:00000010:2.0:1788844870.225683:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000000df8487c. 00000400:00000010:2.0:1788844870.225685:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000009d278723. 00000400:00000010:2.0:1788844870.225687:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000bc3994fc. 00000400:00000010:2.0:1788844870.225690:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b075f539. 00000400:00000010:2.0:1788844870.225692:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ba4b6ecb. 00000400:00000010:2.0:1788844870.225694:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000003ba48064. 00000400:00000010:2.0:1788844870.225695:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000737da499. 00000400:00000010:2.0:1788844870.225698:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ac6fdf8f. 00000400:00000010:2.0:1788844870.225701:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a9e95ace. 00000400:00000010:2.0:1788844870.225702:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000f2fedfc5. 00000400:00000010:2.0:1788844870.225704:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000654bf4b2. 00000400:00000010:2.0:1788844870.225712:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000085d84fe0. 00000400:00000010:2.0:1788844870.225714:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000081355940. 00000400:00000010:2.0:1788844870.225716:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d89ff0c9. 00000400:00000010:2.0:1788844870.225718:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000042b64368. 00000400:00000010:2.0:1788844870.225721:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000666acba5. 00000400:00000010:2.0:1788844870.225724:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000d55bb4a0. 00000400:00000010:2.0:1788844870.225725:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000060ec8392. 00000400:00000010:2.0:1788844870.225727:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001b4c5c26. 00000400:00000010:2.0:1788844870.225730:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f2896859. 00000400:00000010:2.0:1788844870.225732:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000d3a31ee7. 00000400:00000010:2.0:1788844870.225734:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000034e6b041. 00000400:00000010:2.0:1788844870.225735:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002e537f08. 00000400:00000010:2.0:1788844870.225739:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001dc6f449. 00000400:00000010:2.0:1788844870.225741:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000014b7cb64. 00000400:00000010:2.0:1788844870.225743:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000552b0ef8. 00000400:00000010:2.0:1788844870.225744:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000140c6969. 00000400:00000010:2.0:1788844870.225752:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ce632e44. 00000400:00000010:2.0:1788844870.225755:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000009829b257. 00000400:00000010:2.0:1788844870.225757:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000de5831f8. 00000400:00000010:2.0:1788844870.225762:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009a9d7211. 00000400:00000010:2.0:1788844870.225766:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000365db3e4. 00000400:00000010:2.0:1788844870.225769:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000734994e0. 00000400:00000010:2.0:1788844870.225770:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000072e0489. 00000400:00000010:2.0:1788844870.225772:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b623ff4b. 00000400:00000010:2.0:1788844870.225775:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000026a45d92. 00000400:00000010:2.0:1788844870.225809:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000d99c3fdc. 00000400:00000010:2.0:1788844870.225813:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d0417543. 00000400:00000010:2.0:1788844870.225815:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008b26e223. 00000400:00000010:2.0:1788844870.225819:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000dd0c6b59. 00000400:00000010:2.0:1788844870.225821:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000008135f41e. 00000400:00000010:2.0:1788844870.226159:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002bfd2633. 00000400:00000010:2.0:1788844870.226165:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000058d09cc6. 00000400:00000010:2.0:1788844870.226177:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a5ec57bb. 00000400:00000010:2.0:1788844870.226180:0:68218:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000024385f93. 00000400:00000010:2.0:1788844870.226182:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000da489f6e. 00000400:00000010:2.0:1788844870.226184:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a9e2ed54. 00000400:00000010:2.0:1788844870.226188:0:68218:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000afe01e78. 00000400:00000010:2.0:1788844870.226191:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005f6fd0eb. 00000400:00000010:2.0:1788844870.226194:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000070a783f. 00000400:00000010:2.0:1788844870.226198:0:68218:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 000000001bd7867a. 00000400:00000010:2.0:1788844870.226202:0:68218:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 560 at 00000000d53387a7. 00000400:00000010:2.0:1788844870.226204:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000007457c726. 00000400:00000010:2.0:1788844870.226206:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000aa794c31. 00000400:00000010:2.0:1788844870.226207:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a79b3e1b. 00000400:00000010:2.0:1788844870.226209:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000cb3d0ee1. 00000400:00000010:2.0:1788844870.226223:0:68218:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 41 at 00000000dd9465a4. 00000400:00000010:2.0:1788844870.226228:0:68218:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000de6b22f4. 00000400:00000010:2.0:1788844870.226241:0:68218:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ea7be958. 00000400:00000200:2.0:1788844870.226251:0:68218:0:(lib-ptl.c:918:LNetSetLazyPortal()) Setting portal 0 lazy 00000400:00000010:2.0:1788844870.226254:0:68218:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000ac4db73a. 00000400:00000010:2.0:1788844870.226257:0:68218:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e4c04f19. 00000400:00000010:2.0:1788844870.226259:0:68218:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000091a2be88. 00000400:00000200:2.0:1788844870.226265:0:68218:0:(api-ni.c:2383:lnet_push_target_post()) posted push target 00000000ac4db73a 00000400:00000200:2.0:1788844870.226267:0:68218:0:(api-ni.c:2340:lnet_push_target_resize()) nbytes 272 success 00000400:00000010:2.0:1788844870.226269:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a4ae4cda. 00000400:00000010:2.0:1788844870.226271:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d5dded7f. 00000400:00000010:2.0:1788844870.226273:0:68218:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000bb78fac6. 00000400:00000010:2.0:1788844870.226275:0:68218:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000008015098. 00000400:00000200:2.0:1788844870.230761:0:68218:0:(peer.c:4457:lnet_peer_discovery_start()) discovery start: 0 00000400:00000200:1.0F:1788844870.232194:0:68220:0:(peer.c:4283:lnet_peer_discovery()) started 00000400:00000010:1.0:1788844870.240308:0:68221:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000003aed2185. 00000400:00000010:1.0:1788844870.240336:0:68221:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000003aed2185. 00000400:00000010:1.0:1788844870.240692:0:68221:0:(api-ni.c:7409:lnet_route_show_start()) alloc '(rlist)': 16 at 0000000068412480. 00000400:00000010:1.0:1788844870.240709:0:68221:0:(api-ni.c:7309:lnet_route_show_done()) kfreed 'rlist': 16 at 0000000068412480. 00000400:00000010:2.0:1788844870.241033:0:68221:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 236 at 0000000085e5a865. 00000400:00000010:2.0:1788844870.241042:0:68221:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 236 at 0000000085e5a865. 00000400:00000010:2.0:1788844870.241303:0:68221:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 0000000035832293. 00000400:00000010:2.0:1788844870.241319:0:68221:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 0000000035832293. 00000400:00000010:2.0:1788844870.241406:0:68221:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 12 at 0000000035832293. 00000400:00000010:2.0:1788844870.241410:0:68221:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 12 at 0000000035832293. 00000400:00000010:2.0:1788844870.245769:0:68221:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 32 at 000000001a8c98e3. 00000001:00000200:2.0:1788844870.245873:0:68221:0:(udsp.c:863:lnet_udsp_get_policy()) Get UDSP at idx = 0 00000001:00000200:2.0:1788844870.245876:0:68221:0:(udsp.c:878:lnet_udsp_get_policy()) Found UDSP (000000007d9b531d) 00000400:00000010:2.0:1788844870.245879:0:68221:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 32 at 000000001a8c98e3. 00000001:02000400:1.0:1788844870.255082:0:68222:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp --if ens2 00000400:00000010:0.0:1788844870.269332:0:68223:0:(api-ni.c:6877:lnet_net_cmd()) alloc '(conf)': 808 at 000000000956d444. 00000400:00000010:0.0:1788844870.269351:0:68223:0:(api-ni.c:6555:lnet_genl_parse_local_ni()) alloc '(tun)': 172 at 000000006e5c1771. 00000400:00000010:0.0:1788844870.269364:0:68223:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 000000006fc81c16. 00000400:00000010:0.0:1788844870.269370:0:68223:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 560 at 00000000faab73a8. 00000400:00000010:0.0:1788844870.269375:0:68223:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000008051bb76. 00000400:00000010:0.0:1788844870.269380:0:68223:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000bc61d396. 00000400:00000010:0.0:1788844870.269382:0:68223:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002210c718. 00000400:00000010:0.0:1788844870.269384:0:68223:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000040cdf5ec. 00000400:00000010:0.0:1788844870.269388:0:68223:0:(config.c:397:lnet_ni_add_interface()) alloc '(ni->ni_interface)': 5 at 0000000012e9100f. 00000400:00000010:0.0:1788844870.269398:0:68223:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002f9f85de. 00000400:00000010:0.0:1788844870.269401:0:68223:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 0000000099036bee. 00000800:00000200:0.0:1788844870.271244:0:68223:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=lo ifindex 1 state 0 ns 000000004dc0f701 00000800:00000200:0.0:1788844870.271253:0:68223:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=lo ifindex 1 state 0 ns 000000004dc0f701 00000800:00000200:0.0:1788844870.271256:0:68223:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=ens2 ifindex 2 state 6 ns 000000004dc0f701 00000800:00000200:0.0:1788844870.271258:0:68223:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=ens2 ifindex 2 state 6 ns 000000004dc0f701 00000800:00000010:0.0:1788844870.271267:0:68223:0:(socklnd.c:2621:ksocknal_startup()) alloc '(net)': 208 at 00000000306fec4d. 00000800:00000200:0.0:1788844870.271281:0:68223:0:(socklnd_modparams.c:295:ksocklnd_lookup_conns_per_peer()) intf ens2 speed -1 00000400:00000200:0.0:1788844870.275292:0:68223:0:(config.c:1657:lnet_get_link_status_locked()) device idx 1 get_link 2 00000400:00000200:0.0:1788844870.275357:0:68223:0:(acceptor.c:209:lnet_acceptor_add_socket()) Added socket on ens2:988 (IP: 192.168.202.29) 00000400:02000000:0.0:1788844870.275371:0:68223:0:(api-ni.c:2823:lnet_startup_lndni()) Added LNI 192.168.202.29@tcp [8/256/0/180] 00000400:00000010:0.0:1788844870.277597:0:68223:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 57 at 00000000c4c8bad6. 00000400:00000010:0.0:1788844870.277611:0:68223:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000009c3b4942. 00000400:00000010:0.0:1788844870.277638:0:68223:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000005a23032a. 00000400:00000010:1.0:1788844870.278426:0:68223:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000de6b22f4. 00000400:00000200:1.0:1788844870.278431:0:68223:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ea7be958 00000400:00000010:1.0:1788844870.278434:0:68223:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000ea7be958. 00000400:00000010:1.0:1788844870.278438:0:68223:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 41 at 00000000dd9465a4. 00000400:00000010:1.0:1788844870.278456:0:68223:0:(api-ni.c:6852:lnet_genl_parse_local_ni()) kfreed 'tun': 172 at 000000006e5c1771. 00000400:00000010:1.0:1788844870.278461:0:68223:0:(api-ni.c:6994:lnet_net_cmd()) kfreed 'conf': 808 at 000000000956d444. 00000400:00000010:1.0:1788844870.322623:0:68238:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000e25e3788. 00000400:00000010:1.0:1788844870.322691:0:68238:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000e25e3788. 00000001:02000400:2.0:1788844870.805497:0:68344:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp1 --if ens2 00000400:00000010:3.0:1788844870.819986:0:68345:0:(api-ni.c:6877:lnet_net_cmd()) alloc '(conf)': 808 at 00000000c5c73382. 00000400:00000010:3.0:1788844870.820008:0:68345:0:(api-ni.c:6555:lnet_genl_parse_local_ni()) alloc '(tun)': 172 at 0000000094fe3740. 00000400:00000010:3.0:1788844870.820020:0:68345:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 00000000f25748dd. 00000400:00000010:3.0:1788844870.820025:0:68345:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 560 at 00000000cb247215. 00000400:00000010:3.0:1788844870.820029:0:68345:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000006a949bd. 00000400:00000010:3.0:1788844870.820034:0:68345:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008334e28c. 00000400:00000010:3.0:1788844870.820036:0:68345:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000099b9809d. 00000400:00000010:3.0:1788844870.820038:0:68345:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000bc812c2a. 00000400:00000010:3.0:1788844870.820042:0:68345:0:(config.c:397:lnet_ni_add_interface()) alloc '(ni->ni_interface)': 5 at 00000000655fc3c2. 00000800:00000010:3.0:1788844870.820052:0:68345:0:(socklnd.c:2621:ksocknal_startup()) alloc '(net)': 208 at 0000000086a58983. 00000800:00000200:3.0:1788844870.820065:0:68345:0:(socklnd_modparams.c:295:ksocklnd_lookup_conns_per_peer()) intf ens2 speed -1 00000400:00000200:3.0:1788844870.820071:0:68345:0:(config.c:1657:lnet_get_link_status_locked()) device idx 1 get_link 2 00000400:00000200:3.0:1788844870.820184:0:68345:0:(acceptor.c:209:lnet_acceptor_add_socket()) Added socket on ens2:988 (IP: 192.168.202.29) 00000400:02000000:3.0:1788844870.820261:0:68345:0:(api-ni.c:2823:lnet_startup_lndni()) Added LNI 192.168.202.29@tcp1 [8/256/0/180] 00000400:00000010:3.0:1788844870.822620:0:68345:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 73 at 000000004c8a6973. 00000400:00000010:3.0:1788844870.822631:0:68345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000cb47952f. 00000400:00000010:3.0:1788844870.822657:0:68345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a90fb858. 00000400:00000010:3.0:1788844870.822672:0:68345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000009c3b4942. 00000400:00000200:3.0:1788844870.822676:0:68345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000005a23032a 00000400:00000010:3.0:1788844870.822678:0:68345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000005a23032a. 00000400:00000010:3.0:1788844870.822682:0:68345:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 57 at 00000000c4c8bad6. 00000400:00000010:3.0:1788844870.822742:0:68345:0:(api-ni.c:6852:lnet_genl_parse_local_ni()) kfreed 'tun': 172 at 0000000094fe3740. 00000400:00000010:3.0:1788844870.822748:0:68345:0:(api-ni.c:6994:lnet_net_cmd()) kfreed 'conf': 808 at 00000000c5c73382. 00000400:00000010:0.0:1788844870.883544:0:68350:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000000487ce74. 00000400:00000010:0.0:1788844870.883615:0:68350:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000000487ce74. 00000400:00000010:0.0:1788844871.783336:0:68468:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000000487ce74. 00000400:00000010:0.0:1788844871.783438:0:68468:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000000487ce74. 00000001:02000400:1.0:1788844871.817549:0:68470:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl discover 192.168.202.29@tcp 00000400:00000010:0.0:1788844871.840494:0:68471:0:(peer.c:230:lnet_peer_alloc()) alloc '(lp)': 360 at 0000000031654556. 00000400:00000200:0.0:1788844871.840511:0:68471:0:(peer.c:266:lnet_peer_alloc()) 0000000031654556 nid 192.168.202.29@tcp 00000400:00000010:0.0:1788844871.840515:0:68471:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 00000000bc34c189. 00000400:00000200:0.0:1788844871.840518:0:68471:0:(peer.c:203:lnet_peer_net_alloc()) 00000000bc34c189 net tcp 00000400:00000010:0.0:1788844871.840521:0:68471:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 00000000acd8e4cb. 00000400:00000200:0.0:1788844871.840525:0:68471:0:(peer.c:184:lnet_peer_ni_alloc()) 00000000acd8e4cb nid 192.168.202.29@tcp 00000400:00000200:0.0:1788844871.840531:0:68471:0:(peer.c:1804:lnet_peer_attach_peer_ni()) peer 192.168.202.29@tcp NID 192.168.202.29@tcp flags 0x0 00000400:00000200:0.0:1788844871.840534:0:68471:0:(peer.c:2183:lnet_peer_ni_traffic_add()) peer 192.168.202.29@tcp: 0 00000400:00000200:0.0:1788844871.840608:0:68471:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.202.29@tcp: 0 00000400:00000200:0.0:1788844871.840610:0:68471:0:(peer.c:2935:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1788844871.841230:0:68220:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1788844871.841238:0:68220:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.202.29@tcp(0000000031654556) state 0x40 00000400:00000010:0.0:1788844871.841311:0:68220:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000032d8b5b0. 00000400:00000010:0.0:1788844871.841317:0:68220:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e3963c1b. 00000400:00000010:0.0:1788844871.841342:0:68220:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000d7eec911. 00000400:00000200:0.0:1788844871.841348:0:68220:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.29@tcp 00000400:00000200:0.0:1788844871.841360:0:68220:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.29@tcp local destination 00000400:00000200:0.0:1788844871.841366:0:68220:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.29@tcp 00000400:00000200:0.0:1788844871.841369:0:68220:0:(lib-move.c:1874:lnet_set_non_mr_pref_nid()) Setting preferred local NID 192.168.202.29@tcp on NMR peer 192.168.202.29@tcp 00000400:00000200:0.0:1788844871.841373:0:68220:0:(peer.c:1171:lnet_peer_ni_set_non_mr_pref_nid()) peer 192.168.202.29@tcp nid 192.168.202.29@tcp: 0 00000400:00000200:0.0:1788844871.841377:0:68220:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [1:1:256:4294967295] 192.168.202.29@tcp LPNI seq info [1:1:8:4294967295] 00000400:00000200:0.0:1788844871.841384:0:68220:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:0.0:1788844871.841392:0:68220:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : GET try# 0 00000800:00000200:0.0:1788844871.841398:0:68220:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.29@tcp 00000800:00000010:0.0:1788844871.841403:0:68220:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000905b4e5b. 00000800:00000010:0.0:1788844871.841412:0:68220:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 0000000065d6e929. 00000800:00000010:0.0:1788844871.841415:0:68220:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 00000000ca3c6de5. 00000800:00000200:0.0:1788844871.841420:0:68220:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000065d6e929] -> 12345-192.168.202.29@tcp (2) 00000800:00000200:0.1F:1788844871.841424:0:68220:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000065d6e929] -> 12345-192.168.202.29@tcp (2) 00000400:00000200:0.0:1788844871.841485:0:68220:0:(peer.c:4022:lnet_peer_send_ping()) peer 192.168.202.29@tcp 00000400:00000200:0.0:1788844871.841487:0:68220:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.202.29@tcp(0000000031654556) state 0x240 rc 0 00000800:00000010:3.0:1788844871.842128:0:68224:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 000000004d481902. 00000800:00000010:3.0:1788844871.842134:0:68224:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 00000000cafeeace. 00000800:00000010:3.0:1788844871.842139:0:68224:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 0000000083415f8f. 00000400:00000200:0.0:1788844871.842238:0:68232:0:(acceptor.c:578:lnet_acceptor()) Accepted connection on ens2:988 00000400:00000200:0.0:1788844871.842259:0:68232:0:(acceptor.c:502:lnet_accept()) Accept 192.168.202.29@tcp from 255.3.0.2 00000800:00000010:0.0:1788844871.842263:0:68232:0:(socklnd.c:737:ksocknal_accept()) alloc '(cr)': 32 at 0000000032d03404. 00000800:00000010:3.0:1788844871.842415:0:68224:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 0000000083415f8f. 00000800:00000010:0.0:1788844871.842553:0:68225:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 0000000003ba3b00. 00000800:00000010:0.0:1788844871.842562:0:68225:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 0000000050217e31. 00000800:00000010:0.0:1788844871.842577:0:68225:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 000000006e5c1771. 00000800:00000200:0.1:1788844871.842582:0:68225:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000065d6e929] -> 12345-192.168.202.29@tcp (2) 00000800:00000200:0.1:1788844871.842587:0:68225:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.202.29@tcp 000000006e5c1771 deleted 00000800:00000010:0.1:1788844871.842589:0:68225:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 000000006e5c1771. 00000800:00000200:0.1:1788844871.842594:0:68225:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:0.1:1788844871.842601:0:68225:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000200:0.1:1788844871.842605:0:68225:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000905b4e5b type 2, nob 96 niov 1 nkiov 0 00000800:00000200:0.0:1788844871.842610:0:68225:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.202.29@tcp p 3.x 192.168.202.29 -> 192.168.202.29:1023 incarnation:1788844870271269142 sched[0] 00000800:00000010:0.0:1788844871.842615:0:68225:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 0000000018f4d00d. 00000800:00000010:0.0:1788844871.843033:0:68225:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 0000000018f4d00d. 00000800:00000010:0.0:1788844871.843038:0:68225:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 0000000050217e31. 00000800:00000010:0.0:1788844871.843074:0:68225:0:(socklnd_cb.c:2179:ksocknal_connd()) kfreed 'cr': 32 at 0000000032d03404. 00000800:00000200:3.1F:1788844871.843282:0:68224:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:3.0:1788844871.843294:0:68224:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.202.29@tcp p 3.x 192.168.202.29 -> 192.168.202.29:988 incarnation:1788844870271269142 sched[0] 00000800:00000010:3.0:1788844871.843298:0:68224:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 00000000cafeeace. 00000400:00000200:3.0:1788844871.843393:0:68224:0:(lib-socket.c:438:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.202.29:988 00000800:00000010:3.0:1788844871.843568:0:68224:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 000000005636bcd6. 00000800:00000010:3.0:1788844871.843572:0:68224:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 00000000cafeeace. 00000800:00000010:3.0:1788844871.843575:0:68224:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 0000000083415f8f. 00000800:00000010:3.0:1788844871.843608:0:68224:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 0000000083415f8f. 00000400:00000200:2.0:1788844871.843694:0:68232:0:(acceptor.c:578:lnet_acceptor()) Accepted connection on ens2:988 00000400:00000200:2.0:1788844871.843705:0:68232:0:(acceptor.c:502:lnet_accept()) Accept 192.168.202.29@tcp from 254.3.0.2 00000800:00000010:2.0:1788844871.843708:0:68232:0:(socklnd.c:737:ksocknal_accept()) alloc '(cr)': 32 at 00000000e5069e7a. 00000800:00000010:3.0:1788844871.843927:0:68226:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 0000000019bdd1d9. 00000800:00000010:3.0:1788844871.843932:0:68226:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 0000000023524448. 00000800:00000010:3.0:1788844871.843945:0:68226:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 000000007875f2bf. 00000800:00000200:3.1:1788844871.843950:0:68226:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000065d6e929] -> 12345-192.168.202.29@tcp (4) 00000800:00000200:3.1:1788844871.843953:0:68226:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.202.29@tcp 000000007875f2bf deleted 00000800:00000010:3.1:1788844871.843955:0:68226:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 000000007875f2bf. 00000800:00000200:3.1:1788844871.843957:0:68226:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 3, ksnr_connected a ksnr_max_conns 1 00000800:00000200:3.0:1788844871.843961:0:68226:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.202.29@tcp p 3.x 192.168.202.29 -> 192.168.202.29:1022 incarnation:1788844870271269142 sched[0] 00000800:00000010:3.0:1788844871.843966:0:68226:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 0000000083415f8f. 00000800:00000010:3.0:1788844871.844020:0:68226:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 0000000083415f8f. 00000800:00000010:3.0:1788844871.844024:0:68226:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 0000000023524448. 00000800:00000200:2.1F:1788844871.844062:0:68224:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 2, ksnr_connected e ksnr_max_conns 1 00000800:00000200:2.0:1788844871.844073:0:68224:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.202.29@tcp p 3.x 192.168.202.29 -> 192.168.202.29:988 incarnation:1788844870271269142 sched[0] 00000800:00000010:2.0:1788844871.844077:0:68224:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 00000000cafeeace. 00000800:00000200:1.0:1788844871.844348:0:68230:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1788844871.844354:0:68230:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000905b4e5b. 00000400:00000200:1.0:1788844871.844359:0:68230:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1788844871.844365:0:68230:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: GET: OK 00000400:00000200:1.0:1788844871.844370:0:68230:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1788844871.844373:0:68230:0:(peer.c:3241:lnet_discovery_event_send()) Ping Send to 192.168.202.29@tcp: 0 00000800:00000010:3.0:1788844871.844450:0:68226:0:(socklnd_cb.c:2179:ksocknal_connd()) kfreed 'cr': 32 at 00000000e5069e7a. 00000400:00000200:1.0:1788844871.844490:0:68230:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : GET - for me 00000400:00000200:1.0:1788844871.844522:0:68230:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.29@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1788844871.844531:0:68230:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.29@tcp of length 64/272 into md 0xd [1] + 0 00000400:00000200:1.0:1788844871.844544:0:68230:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.29@tcp to NMR: 192.168.202.29@tcp local destination 00000400:00000200:1.0:1788844871.844549:0:68230:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [2:2:256:4294967295] 192.168.202.29@tcp LPNI seq info [2:2:8:4294967295] 00000400:00000200:1.0:1788844871.844559:0:68230:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) 192.168.202.29@tcp : REPLY try# 0 00000800:00000200:1.0:1788844871.844564:0:68230:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.202.29@tcp 00000800:00000010:1.0:1788844871.844569:0:68230:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000006ec8db9. 00000800:00000200:1.0:1788844871.844580:0:68230:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000065d6e929] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:1.0:1788844871.844585:0:68230:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000200:1.0:1788844871.844588:0:68230:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000006ec8db9 type 3, nob 160 niov 1 nkiov 1 00000800:00000200:1.0:1788844871.845031:0:68230:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1788844871.845035:0:68230:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000006ec8db9. 00000400:00000200:1.0:1788844871.845039:0:68230:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1788844871.845043:0:68230:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: REPLY: OK 00000400:00000200:2.0:1788844871.845199:0:68229:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : REPLY - for me 00000400:00000200:2.0:1788844871.845223:0:68229:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.29@tcp: Reply from 12345-192.168.202.29@tcp of length 64/64 into md 0x11 00000400:00000200:2.0:1788844871.845232:0:68229:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788844871.845236:0:68229:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: REPLY: OK 00000400:00000200:2.0:1788844871.845240:0:68229:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e3963c1b 00000400:00000010:2.0:1788844871.845243:0:68229:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000e3963c1b. 00000400:00000200:2.0:1788844871.845246:0:68229:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 3 00000400:00000200:2.0:1788844871.845249:0:68229:0:(peer.c:3119:lnet_discovery_event_reply()) Peer 192.168.202.29@tcp has discovery enabled 00000400:00000200:2.0:1788844871.845252:0:68229:0:(peer.c:3145:lnet_discovery_event_reply()) peer 192.168.202.29@tcp(0000000031654556) is MR capable 00000400:00000200:2.0:1788844871.845255:0:68229:0:(peer.c:1196:lnet_peer_ni_clr_non_mr_pref_nid()) peer 192.168.202.29@tcp: 0 00000400:00000200:2.0:1788844871.845265:0:68229:0:(peer.c:3200:lnet_discovery_event_reply()) peer 192.168.202.29@tcp data present 3. state = 0x241 00000400:00000200:2.0:1788844871.845269:0:68229:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.202.29@tcp: -114 00000400:00000200:3.0:1788844871.845331:0:68220:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1788844871.845336:0:68220:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.202.29@tcp(0000000031654556) state 0xc1 00000400:00000010:3.0:1788844871.845343:0:68220:0:(peer.c:3474:lnet_peer_merge_data()) alloc '(curnis)': 60 at 0000000083415f8f. 00000400:00000010:3.0:1788844871.845347:0:68220:0:(peer.c:3475:lnet_peer_merge_data()) alloc '(addnis)': 72 at 0000000065731974. 00000400:00000010:3.0:1788844871.845349:0:68220:0:(peer.c:3476:lnet_peer_merge_data()) alloc '(delnis)': 60 at 00000000818f8a99. 00000400:00000010:3.0:1788844871.845358:0:68220:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 000000000a51eb25. 00000400:00000200:3.0:1788844871.845362:0:68220:0:(peer.c:184:lnet_peer_ni_alloc()) 000000000a51eb25 nid 192.168.202.29@tcp1 00000400:00000010:3.0:1788844871.845366:0:68220:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 00000000fed799a0. 00000400:00000200:3.0:1788844871.845369:0:68220:0:(peer.c:203:lnet_peer_net_alloc()) 00000000fed799a0 net tcp1 00000400:00000200:3.0:1788844871.845374:0:68220:0:(peer.c:1804:lnet_peer_attach_peer_ni()) peer 192.168.202.29@tcp NID 192.168.202.29@tcp1 flags 0x11 00000400:00000010:3.0:1788844871.845378:0:68220:0:(peer.c:3623:lnet_peer_merge_data()) kfreed 'curnis': 60 at 0000000083415f8f. 00000400:00000010:3.0:1788844871.845380:0:68220:0:(peer.c:3624:lnet_peer_merge_data()) kfreed 'addnis': 72 at 0000000065731974. 00000400:00000010:3.0:1788844871.845383:0:68220:0:(peer.c:3625:lnet_peer_merge_data()) kfreed 'delnis': 60 at 00000000818f8a99. 00000400:00000010:3.0:1788844871.845386:0:68220:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000032d8b5b0. 00000400:00000200:3.0:1788844871.845389:0:68220:0:(peer.c:3628:lnet_peer_merge_data()) peer 192.168.202.29@tcp (0000000031654556): 0 00000400:00000200:3.0:1788844871.845392:0:68220:0:(peer.c:3940:lnet_peer_data_present()) peer 192.168.202.29@tcp(0000000031654556): 0. state = 0x141 00000400:00000200:3.0:1788844871.845395:0:68220:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.202.29@tcp(0000000031654556) state 0x141 rc 1 00000400:00000200:3.0:1788844871.845405:0:68220:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.202.29@tcp(0000000031654556) state 0x141 00000400:00000010:3.0:1788844871.845410:0:68220:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000023d7d902. 00000400:00000010:3.0:1788844871.845426:0:68220:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000e990a3fd. 00000400:00000200:3.0:1788844871.845431:0:68220:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.29@tcp 00000400:00000200:3.0:1788844871.845438:0:68220:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.29@tcp to MR: 192.168.202.29@tcp local destination 00000400:00000200:3.0:1788844871.845443:0:68220:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [3:3:256:4294967295] 192.168.202.29@tcp LPNI seq info [3:3:8:4294967295] 00000400:00000200:3.0:1788844871.845447:0:68220:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:3.0:1788844871.845453:0:68220:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : PUT try# 0 00000800:00000200:3.0:1788844871.845457:0:68220:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.202.29@tcp 00000800:00000010:3.0:1788844871.845460:0:68220:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000007d893bf4. 00000800:00000200:3.0:1788844871.845464:0:68220:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000065d6e929] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:3.0:1788844871.845469:0:68220:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000200:3.0:1788844871.845471:0:68220:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000007d893bf4 type 1, nob 160 niov 1 nkiov 1 00000400:00000200:3.0:1788844871.845476:0:68220:0:(peer.c:4164:lnet_peer_send_push()) peer 192.168.202.29@tcp 00000400:00000200:3.0:1788844871.845478:0:68220:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.202.29@tcp(0000000031654556) state 0x541 rc 0 00000800:00000200:0.0:1788844871.845643:0:68231:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788844871.845646:0:68231:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000007d893bf4. 00000400:00000200:0.0:1788844871.845649:0:68231:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788844871.845653:0:68231:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: PUT: OK 00000400:00000200:0.0:1788844871.845656:0:68231:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1788844871.845658:0:68231:0:(peer.c:3241:lnet_discovery_event_send()) Push Send to 192.168.202.29@tcp: 0 00000400:00000200:0.0:1788844871.845669:0:68231:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : PUT - for me 00000400:00000200:0.0:1788844871.845691:0:68231:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.29@tcp of length 64 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1788844871.845696:0:68231:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 0 from 12345-192.168.202.29@tcp of length 64/64 into md 0x5 [1] + 0 00000400:00000010:0.0:1788844871.845700:0:68231:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000e4c04f19. 00000400:00000200:0.0:1788844871.845704:0:68231:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000091a2be88 00000400:00000200:0.0:1788844871.845710:0:68231:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788844871.845714:0:68231:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: PUT: OK 00000400:00000200:0.0:1788844871.845716:0:68231:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000091a2be88 00000400:00000010:0.0:1788844871.845717:0:68231:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 0000000091a2be88. 00000400:00000200:0.0:1788844871.845721:0:68231:0:(api-ni.c:2393:lnet_push_target_event_handler()) type 2 status 0 unlinked 1 00000400:00000200:0.0:1788844871.845726:0:68231:0:(peer.c:2778:lnet_peer_push_event()) peer 192.168.202.29@tcp(0000000031654556) is MR 00000400:00000010:0.0:1788844871.845731:0:68231:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 73 at 0000000026ed2ce0. 00000400:00000200:0.0:1788844871.845734:0:68231:0:(peer.c:2852:lnet_peer_push_event()) Received Push 192.168.202.29@tcp 3 00000400:00000200:0.0:1788844871.845736:0:68231:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.202.29@tcp: -114 00000400:00000200:0.0:1788844871.845750:0:68231:0:(lib-md.c:96:lnet_cpt_of_md()) Couldn't resolve first page of md 0000000000000000 with offset 0 00000400:00000200:0.0:1788844871.845754:0:68231:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.29@tcp to MR: 192.168.202.29@tcp local destination 00000400:00000200:0.0:1788844871.845757:0:68231:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [4:4:256:4294967295] 192.168.202.29@tcp LPNI seq info [4:4:8:4294967295] 00000400:00000200:0.0:1788844871.845764:0:68231:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) 192.168.202.29@tcp : ACK try# 0 00000800:00000200:0.0:1788844871.845775:0:68231:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 1 frags to 12345-192.168.202.29@tcp 00000800:00000010:0.0:1788844871.845912:0:68231:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000020e2fd8c. 00000800:00000200:0.0:1788844871.845917:0:68231:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000065d6e929] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:0.0:1788844871.845921:0:68231:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000200:0.0:1788844871.845924:0:68231:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000020e2fd8c type 0, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1788844871.845964:0:68220:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000010:3.0:1788844871.845969:0:68220:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000009e812ea7. 00000400:00000010:3.0:1788844871.845973:0:68220:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004b62fb26. 00000400:00000200:3.0:1788844871.845977:0:68220:0:(api-ni.c:2383:lnet_push_target_post()) posted push target 00000000ac4db73a 00000400:00000200:3.0:1788844871.845981:0:68220:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.202.29@tcp(0000000031654556) state 0x4c1 00000400:00000010:3.0:1788844871.845985:0:68220:0:(peer.c:3474:lnet_peer_merge_data()) alloc '(curnis)': 60 at 00000000818f8a99. 00000400:00000010:3.0:1788844871.845988:0:68220:0:(peer.c:3475:lnet_peer_merge_data()) alloc '(addnis)': 72 at 0000000067c24a58. 00000400:00000010:3.0:1788844871.845990:0:68220:0:(peer.c:3476:lnet_peer_merge_data()) alloc '(delnis)': 60 at 0000000083415f8f. 00000400:00000010:3.0:1788844871.845993:0:68220:0:(peer.c:3623:lnet_peer_merge_data()) kfreed 'curnis': 60 at 00000000818f8a99. 00000400:00000010:3.0:1788844871.845995:0:68220:0:(peer.c:3624:lnet_peer_merge_data()) kfreed 'addnis': 72 at 0000000067c24a58. 00000400:00000010:3.0:1788844871.845997:0:68220:0:(peer.c:3625:lnet_peer_merge_data()) kfreed 'delnis': 60 at 0000000083415f8f. 00000400:00000010:3.0:1788844871.846000:0:68220:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 73 at 0000000026ed2ce0. 00000400:00000200:3.0:1788844871.846003:0:68220:0:(peer.c:3628:lnet_peer_merge_data()) peer 192.168.202.29@tcp (0000000031654556): 0 00000400:00000200:3.0:1788844871.846004:0:68220:0:(peer.c:3940:lnet_peer_data_present()) peer 192.168.202.29@tcp(0000000031654556): 0. state = 0x541 00000400:00000200:3.0:1788844871.846007:0:68220:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.202.29@tcp(0000000031654556) state 0x541 rc 1 00000400:00000200:3.0:1788844871.846010:0:68220:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.202.29@tcp(0000000031654556) state 0x541 00000400:00000200:3.0:1788844871.846018:0:68220:0:(peer.c:4082:lnet_peer_discovered()) peer 192.168.202.29@tcp 00000400:00000200:3.0:1788844871.846020:0:68220:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.202.29@tcp(0000000031654556) state 0x511 rc 0 00000400:00000200:3.0:1788844871.846024:0:68220:0:(peer.c:2629:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 192.168.202.29@tcp 00000400:00000200:1.0:1788844871.846372:0:68471:0:(peer.c:2984:lnet_discover_peer_locked()) peer 192.168.202.29@tcp NID 192.168.202.29@tcp: 0. discovery complete 00000800:00000200:1.0:1788844871.853396:0:68230:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1788844871.853405:0:68230:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000020e2fd8c. 00000400:00000200:1.0:1788844871.853411:0:68230:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1788844871.853419:0:68230:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: ACK: OK 00000400:00000200:1.0:1788844871.853444:0:68230:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : ACK - for me 00000400:00000200:1.0:1788844871.853458:0:68230:0:(lib-move.c:4539:lnet_parse_ack()) 192.168.202.29@tcp: ACK from 12345-192.168.202.29@tcp into md 0x15 00000400:00000200:1.0:1788844871.853464:0:68230:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1788844871.853467:0:68230:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: ACK: OK 00000400:00000200:1.0:1788844871.853472:0:68230:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000023d7d902 00000400:00000010:1.0:1788844871.853475:0:68230:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 0000000023d7d902. 00000400:00000200:1.0:1788844871.853479:0:68230:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 4 00000400:00000200:1.0:1788844871.853482:0:68230:0:(peer.c:3007:lnet_discovery_event_ack()) peer 192.168.202.29@tcp ev->status 0 00000400:00000010:3.0:1788844871.867561:0:68473:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 12 at 00000000d192559b. 00000400:00000010:3.0:1788844871.867576:0:68473:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 12 at 00000000d192559b. 00000001:02000400:2.0:1788844871.924391:0:68475:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 10 00000001:02000400:1.0:1788844871.970230:0:68478:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net_drop add -s *@tcp -d *@tcp -m GET -r 1 -e local_error 00000400:00000200:2.0:1788844871.990603:0:68479:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844871.990615:0:68479:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844871.990627:0:68479:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp 00000400:00000200:2.0:1788844871.990629:0:68479:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844871.990632:0:68479:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp 00000400:00000200:2.0:1788844871.990633:0:68479:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844871.990635:0:68479:0:(api-ni.c:3207:nla_strnid()) -> 00000400:00000200:2.0:1788844871.990636:0:68479:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844871.990637:0:68479:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844871.990638:0:68479:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844871.990640:0:68479:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844871.990641:0:68479:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844871.990643:0:68479:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844871.990645:0:68479:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000010:2.0:1788844871.990649:0:68479:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 00000000d558236e. 00000400:00000200:2.0:1788844871.990657:0:68479:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp, dst 255.255.255.255@tcp, rate 1, interval 0 00000001:02000400:2.0:1788844872.000250:0:68480:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net_drop add -s *@tcp1 -d *@tcp1 -m GET -r 1 -e local_error 00000400:00000200:2.0:1788844872.016343:0:68481:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844872.016355:0:68481:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844872.016371:0:68481:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1 00000400:00000200:2.0:1788844872.016373:0:68481:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844872.016377:0:68481:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1 00000400:00000200:2.0:1788844872.016378:0:68481:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844872.016381:0:68481:0:(api-ni.c:3207:nla_strnid()) -> 00000400:00000200:2.0:1788844872.016382:0:68481:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844872.016383:0:68481:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844872.016384:0:68481:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844872.016386:0:68481:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844872.016388:0:68481:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844872.016389:0:68481:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844872.016391:0:68481:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000010:2.0:1788844872.016395:0:68481:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 0000000043c070b4. 00000400:00000200:2.0:1788844872.016403:0:68481:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp1, dst 255.255.255.255@tcp1, rate 1, interval 0 00000001:02000400:1.0:1788844872.024943:0:68482:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net_drop add -s *@tcp -d *@tcp -r 1 00000400:00000200:0.0:1788844872.041196:0:68483:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844872.041206:0:68483:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844872.041218:0:68483:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp 00000400:00000200:0.0:1788844872.041220:0:68483:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844872.041222:0:68483:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp 00000400:00000200:0.0:1788844872.041223:0:68483:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844872.041225:0:68483:0:(api-ni.c:3207:nla_strnid()) -> 00000400:00000200:0.0:1788844872.041227:0:68483:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844872.041228:0:68483:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844872.041229:0:68483:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844872.041231:0:68483:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844872.041232:0:68483:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844872.041234:0:68483:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844872.041236:0:68483:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000010:0.0:1788844872.041240:0:68483:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 00000000cfb2d9af. 00000400:00000200:0.0:1788844872.041248:0:68483:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp, dst 255.255.255.255@tcp, rate 1, interval 0 00000001:02000400:0.0:1788844872.050562:0:68484:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp1 -d *@tcp1 -r 1 00000400:00000200:0.0:1788844872.065634:0:68485:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844872.065646:0:68485:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844872.065660:0:68485:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1 00000400:00000200:0.0:1788844872.065663:0:68485:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844872.065666:0:68485:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1 00000400:00000200:0.0:1788844872.065667:0:68485:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844872.065670:0:68485:0:(api-ni.c:3207:nla_strnid()) -> 00000400:00000200:0.0:1788844872.065671:0:68485:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844872.065672:0:68485:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844872.065674:0:68485:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844872.065675:0:68485:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844872.065677:0:68485:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844872.065679:0:68485:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844872.065680:0:68485:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000010:0.0:1788844872.065685:0:68485:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 0000000032d8b5b0. 00000400:00000200:0.0:1788844872.065693:0:68485:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp1, dst 255.255.255.255@tcp1, rate 1, interval 0 00000001:02000400:3.0:1788844872.075747:0:68486:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net set --health 0 --nid 192.168.202.29@tcp 00000400:00000010:1.0:1788844872.094173:0:68487:0:(api-ni.c:6877:lnet_net_cmd()) alloc '(conf)': 808 at 00000000d30daf99. 00000400:00000010:1.0:1788844872.094193:0:68487:0:(api-ni.c:6555:lnet_genl_parse_local_ni()) alloc '(tun)': 172 at 00000000d8a5d020. 00000400:00000200:1.0:1788844872.094199:0:68487:0:(api-ni.c:6621:lnet_genl_parse_local_ni()) Requested NID 192.168.202.29@tcp 00000400:00020000:1.0:1788844872.094215:0:68487:0:(api-ni.c:4473:lnet_ni_set_healthv()) manually adding local NI 192.168.202.29@tcp to recovery 00000400:00000200:1.0:1788844872.099949:0:68487:0:(lib-lnet.h:200:lnet_ni_set_status_locked()) ni 192.168.202.29@tcp status changed from 0x15aac0de to 0xdeadface 00000400:00000010:1.0:1788844872.099958:0:68487:0:(api-ni.c:6852:lnet_genl_parse_local_ni()) kfreed 'tun': 172 at 00000000d8a5d020. 00000400:00000010:1.0:1788844872.099967:0:68487:0:(api-ni.c:6994:lnet_net_cmd()) kfreed 'conf': 808 at 00000000d30daf99. 00000400:00000010:0.0:1788844872.155521:0:68493:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000000487ce74. 00000400:00000010:0.0:1788844872.155640:0:68493:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000000487ce74. 00000400:00000200:3.0:1788844872.280076:0:68219:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.29@tcp 00000400:00000010:3.0:1788844872.280149:0:68219:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000c173b669. 00000400:00000010:3.0:1788844872.280160:0:68219:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000f4e70950. 00000400:00000010:3.0:1788844872.280166:0:68219:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000acd7827e. 00000400:00000010:3.0:1788844872.280175:0:68219:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000003f3c6d43. 00000400:00000200:3.0:1788844872.280179:0:68219:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.29@tcp 00000400:00000200:3.0:1788844872.280194:0:68219:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.29@tcp local destination 00000400:00000200:3.0:1788844872.280200:0:68219:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.29@tcp 00000400:00000200:3.0:1788844872.280204:0:68219:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [5:5:256:4294967295] 192.168.202.29@tcp LPNI seq info [5:5:8:4294967295] 00000400:00000200:3.0:1788844872.280210:0:68219:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:3.0:1788844872.280218:0:68219:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : GET try# 0 00000800:00000200:3.0:1788844872.280223:0:68219:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.29@tcp 00000800:00000010:3.0:1788844872.280228:0:68219:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000995351cb. 00000800:00000200:3.0:1788844872.280234:0:68219:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000065d6e929] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:3.0:1788844872.280241:0:68219:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000200:3.0:1788844872.280245:0:68219:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000995351cb type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1788844872.280324:0:68219:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.29@tcp added to recovery queue. ping count: 1 next ping: 1438 health :0 00000400:00000010:3.0:1788844872.280331:0:68219:0:(lib-move.c:3814:lnet_print_recovery_list()) alloc '(buf)': 65 at 0000000067c24a58. 00000400:02000040:3.0:1788844872.280335:0:68219:0:(lib-move.c:3830:lnet_print_recovery_list()) 1 local NIs in recovery (showing 1): 192.168.202.29@tcp 00000400:00000200:1.0:1788844872.281066:0:68230:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 1 00000400:00000200:1.0:1788844872.281075:0:68230:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.202.29@tcp(192.168.202.29@tcp)->dst 192.168.202.29@tcp: GET simulate health error: LOCAL_ERROR 00000800:02000400:1.0:1788844872.281104:0:68230:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.202.29: rc = -22 00000400:00000010:3.0:1788844872.283908:0:68219:0:(lib-move.c:3832:lnet_print_recovery_list()) kfreed 'buf': 65 at 0000000067c24a58. 00000800:00000200:1.0:1788844872.287637:0:68230:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [000000004d481902] Error -22 on write to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000010:1.0:1788844872.287656:0:68230:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000995351cb. 00000400:00000200:1.0:1788844872.287663:0:68230:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 5 00000400:00000200:1.0:1788844872.287667:0:68230:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: GET: LOCAL_ERROR 00000400:00000200:1.0:1788844872.287677:0:68230:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5 00000400:00000200:1.0:1788844872.287681:0:68230:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.29@tcp recovery message sent unsuccessfully:-5 00000400:00000200:1.0:1788844872.287684:0:68230:0:(lib-move.c:4071:lnet_handle_recovery_reply()) local NI (192.168.202.29@tcp) recovery failed with -5 00000400:00000010:3.0:1788844873.242060:0:68503:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000bc565e63. 00000400:00000010:3.0:1788844873.242170:0:68503:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000bc565e63. 00000400:00000200:3.0:1788844873.304170:0:68219:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000acd7827e 00000400:00000010:3.0:1788844873.304181:0:68219:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000acd7827e. 00000400:00000200:3.0:1788844873.304211:0:68219:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0 00000400:00000200:3.0:1788844873.304218:0:68219:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.202.29@tcp recovery ping unlinked 00000400:00000010:3.0:1788844873.304222:0:68219:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000c173b669. 00000400:00000010:3.0:1788844873.304227:0:68219:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000f4e70950. 00000400:00000200:3.0:1788844874.335966:0:68219:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.29@tcp 00000400:00000010:3.0:1788844874.335976:0:68219:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000033014b15. 00000400:00000010:3.0:1788844874.335986:0:68219:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004440e37d. 00000400:00000010:3.0:1788844874.335992:0:68219:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000acd7827e. 00000400:00000010:3.0:1788844874.336001:0:68219:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000d69ff3ea. 00000400:00000200:3.0:1788844874.336005:0:68219:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.29@tcp 00000400:00000200:3.0:1788844874.336021:0:68219:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.29@tcp local destination 00000400:00000200:3.0:1788844874.336029:0:68219:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.29@tcp 00000400:00000200:3.0:1788844874.336032:0:68219:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [6:6:256:4294967295] 192.168.202.29@tcp LPNI seq info [6:6:8:4294967295] 00000400:00000200:3.0:1788844874.336039:0:68219:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:3.0:1788844874.336046:0:68219:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : GET try# 0 00000800:00000200:3.0:1788844874.336052:0:68219:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.29@tcp 00000800:00000010:3.0:1788844874.336059:0:68219:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000b88ac0e1. 00000800:00000200:3.0:1788844874.336066:0:68219:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000065d6e929] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:3.0:1788844874.336076:0:68219:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000200:3.0:1788844874.336081:0:68219:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000b88ac0e1 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1788844874.336096:0:68219:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.29@tcp added to recovery queue. ping count: 2 next ping: 1442 health :0 00000400:00000200:3.0:1788844874.336125:0:68229:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 2 00000400:00000200:3.0:1788844874.336130:0:68229:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.202.29@tcp(192.168.202.29@tcp)->dst 192.168.202.29@tcp: GET simulate health error: LOCAL_ERROR 00000800:02000400:3.0:1788844874.336133:0:68229:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.202.29: rc = -22 00000800:00000200:3.0:1788844874.342491:0:68229:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [0000000003ba3b00] Error -22 on write to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000010:3.0:1788844874.342508:0:68229:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000b88ac0e1. 00000400:00000200:3.0:1788844874.342514:0:68229:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 5 00000400:00000200:3.0:1788844874.342521:0:68229:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: GET: LOCAL_ERROR 00000400:00000200:3.0:1788844874.342534:0:68229:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5 00000400:00000200:3.0:1788844874.342554:0:68229:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.29@tcp recovery message sent unsuccessfully:-5 00000400:00000200:3.0:1788844874.342559:0:68229:0:(lib-move.c:4071:lnet_handle_recovery_reply()) local NI (192.168.202.29@tcp) recovery failed with -5 00000400:00000010:0.0:1788844874.347207:0:68513:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000000487ce74. 00000400:00000010:0.0:1788844874.347323:0:68513:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000000487ce74. 00000400:00000010:2.0:1788844874.399449:0:68517:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000fc6c79b4. 00000400:00000010:2.0:1788844874.399610:0:68517:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000fc6c79b4. 00000400:00000200:3.0:1788844874.493824:0:68524:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 0 00000400:00000010:3.0:1788844874.493837:0:68524:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000009cb06545. 00000400:00000200:3.0:1788844874.493852:0:68524:0:(api-ni.c:9567:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:3.0:1788844874.493876:0:68524:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000608735b5. 00000400:00000200:3.0:1788844874.493884:0:68524:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:3.0:1788844874.493889:0:68524:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000608735b5. 00000400:00000010:3.0:1788844874.493898:0:68524:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000009cb06545. 00000001:02000400:2.0:1788844874.535896:0:68526:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -l 00000400:00000200:3.0:1788844874.558950:0:68527:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 0 00000400:00000010:3.0:1788844874.559138:0:68527:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000009cb06545. 00000400:00000200:3.0:1788844874.559153:0:68527:0:(api-ni.c:9567:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:3.0:1788844874.559178:0:68527:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000608735b5. 00000400:00000200:3.0:1788844874.559188:0:68527:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:3.0:1788844874.559193:0:68527:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000608735b5. 00000400:00000010:3.0:1788844874.559202:0:68527:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000009cb06545. 00000400:00000010:1.0:1788844874.666991:0:68533:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000003aed2185. 00000400:00000010:1.0:1788844874.667510:0:68533:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000003aed2185. 00000400:00000200:3.0:1788844875.351981:0:68219:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000acd7827e 00000400:00000010:3.0:1788844875.351999:0:68219:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000acd7827e. 00000400:00000200:3.0:1788844875.352009:0:68219:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0 00000400:00000200:3.0:1788844875.352019:0:68219:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.202.29@tcp recovery ping unlinked 00000400:00000010:3.0:1788844875.352024:0:68219:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000033014b15. 00000400:00000010:3.0:1788844875.352044:0:68219:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004440e37d. 00000400:00000010:1.0:1788844875.807814:0:68544:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000003aed2185. 00000400:00000010:1.0:1788844875.807944:0:68544:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000003aed2185. 00000400:00000010:2.0:1788844876.934088:0:68554:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000fc6c79b4. 00000400:00000010:2.0:1788844876.934237:0:68554:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000fc6c79b4. 00000400:00000010:1.0:1788844878.080372:0:68564:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000003aed2185. 00000400:00000010:1.0:1788844878.083255:0:68564:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000003aed2185. 00000400:00000200:3.0:1788844878.424632:0:68219:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.29@tcp 00000400:00000010:3.0:1788844878.424648:0:68219:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000233100cf. 00000400:00000010:3.0:1788844878.424660:0:68219:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000f4e70950. 00000400:00000010:3.0:1788844878.424667:0:68219:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000acd7827e. 00000400:00000010:3.0:1788844878.424711:0:68219:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000027f265b9. 00000400:00000200:3.0:1788844878.424717:0:68219:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.29@tcp 00000400:00000200:3.0:1788844878.424736:0:68219:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.29@tcp local destination 00000400:00000200:3.0:1788844878.424743:0:68219:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.29@tcp 00000400:00000200:3.0:1788844878.424747:0:68219:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [7:7:256:4294967295] 192.168.202.29@tcp LPNI seq info [7:7:8:4294967295] 00000400:00000200:3.0:1788844878.424810:0:68219:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:3.0:1788844878.424825:0:68219:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : GET try# 0 00000800:00000200:3.0:1788844878.424832:0:68219:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.29@tcp 00000800:00000010:3.0:1788844878.424837:0:68219:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000b88ac0e1. 00000800:00000200:3.0:1788844878.424845:0:68219:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000065d6e929] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:3.0:1788844878.424853:0:68219:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000200:3.0:1788844878.424858:0:68219:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000b88ac0e1 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1788844878.424880:0:68219:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.29@tcp added to recovery queue. ping count: 3 next ping: 1450 health :0 00000400:00000200:0.0:1788844878.425288:0:68231:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 3 00000400:00000200:0.0:1788844878.425294:0:68231:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.202.29@tcp(192.168.202.29@tcp)->dst 192.168.202.29@tcp: GET simulate health error: LOCAL_ERROR 00000800:02000400:0.0:1788844878.425296:0:68231:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.202.29: rc = -22 00000800:00000200:0.0:1788844878.429311:0:68231:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [000000004d481902] Error -22 on write to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000010:0.0:1788844878.429323:0:68231:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000b88ac0e1. 00000400:00000200:0.0:1788844878.429333:0:68231:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 5 00000400:00000200:0.0:1788844878.429338:0:68231:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: GET: LOCAL_ERROR 00000400:00000200:0.0:1788844878.429349:0:68231:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5 00000400:00000200:0.0:1788844878.429352:0:68231:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.29@tcp recovery message sent unsuccessfully:-5 00000400:00000200:0.0:1788844878.429354:0:68231:0:(lib-move.c:4071:lnet_handle_recovery_reply()) local NI (192.168.202.29@tcp) recovery failed with -5 00000400:00000010:1.0:1788844879.256088:0:68574:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000003aed2185. 00000400:00000010:1.0:1788844879.256224:0:68574:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000003aed2185. 00000400:00000010:1.0:1788844879.359845:0:68578:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000003aed2185. 00000400:00000010:1.0:1788844879.359989:0:68578:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000003aed2185. 00000400:00000200:3.0:1788844879.448572:0:68219:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000acd7827e 00000400:00000010:3.0:1788844879.448588:0:68219:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000acd7827e. 00000400:00000200:3.0:1788844879.448596:0:68219:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0 00000400:00000200:3.0:1788844879.448605:0:68219:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.202.29@tcp recovery ping unlinked 00000400:00000010:3.0:1788844879.448609:0:68219:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000233100cf. 00000400:00000010:3.0:1788844879.448616:0:68219:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000f4e70950. 00000400:00000200:3.0:1788844879.489708:0:68585:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 0 00000400:00000010:3.0:1788844879.489722:0:68585:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000003a74fcc1. 00000400:00000200:3.0:1788844879.489736:0:68585:0:(api-ni.c:9567:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:3.0:1788844879.489773:0:68585:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000b083c074. 00000400:00000200:3.0:1788844879.489810:0:68585:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:3.0:1788844879.489816:0:68585:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000b083c074. 00000400:00000010:3.0:1788844879.489824:0:68585:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000003a74fcc1. 00000001:02000400:3.0:1788844879.524083:0:68587:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -l 00000400:00000200:2.0:1788844879.561224:0:68588:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 0 00000400:00000010:2.0:1788844879.561239:0:68588:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000002979f328. 00000400:00000200:2.0:1788844879.561253:0:68588:0:(api-ni.c:9567:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:2.0:1788844879.561276:0:68588:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000137653d3. 00000400:00000200:2.0:1788844879.561284:0:68588:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:2.0:1788844879.561288:0:68588:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000137653d3. 00000400:00000010:2.0:1788844879.561300:0:68588:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000002979f328. 00000001:02000400:3.0:1788844879.610659:0:68589:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net_drop del -a 00000400:00000200:2.0:1788844879.628962:0:68590:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844879.628977:0:68590:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844879.628983:0:68590:0:(api-ni.c:3207:nla_strnid()) -> 00000400:00000200:2.0:1788844879.628985:0:68590:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844879.628987:0:68590:0:(api-ni.c:3207:nla_strnid()) -> 00000400:00000200:2.0:1788844879.628990:0:68590:0:(net_fault.c:254:lnet_drop_rule_del()) src dst 00000400:00000200:2.0:1788844879.629002:0:68590:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp->dst: 255.255.255.255@tcp (1/1, 0) 00000400:00000010:2.0:1788844879.629004:0:68590:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 00000000d558236e. 00000400:00000200:2.0:1788844879.629011:0:68590:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp1->dst: 255.255.255.255@tcp1 (1/1, 0) 00000400:00000010:2.0:1788844879.629013:0:68590:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 0000000043c070b4. 00000400:00000200:2.0:1788844879.629017:0:68590:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp->dst: 255.255.255.255@tcp (1/1, 0) 00000400:00000010:2.0:1788844879.629088:0:68590:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 00000000cfb2d9af. 00000400:00000200:2.0:1788844879.629096:0:68590:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp1->dst: 255.255.255.255@tcp1 (1/1, 0) 00000400:00000010:2.0:1788844879.629097:0:68590:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 0000000032d8b5b0. 00000001:02000400:0.0:1788844879.661036:0:68593:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure 00000400:00000200:1.0:1788844879.682975:0:68594:0:(net_fault.c:254:lnet_drop_rule_del()) src dst 00000400:00000200:0.0:1788844879.683776:0:68220:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: -108 00000400:00000200:0.0:1788844879.683934:0:68220:0:(peer.c:4385:lnet_peer_discovery()) stopping 00000400:00000200:0.0:1788844879.683937:0:68220:0:(peer.c:4431:lnet_peer_discovery()) stopped 00000400:00000200:1.0:1788844879.692965:0:68594:0:(peer.c:4488:lnet_peer_discovery_stop()) discovery stopped 00000400:00000010:1.0:1788844879.701975:0:68594:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 00000000d7eec911. 00000400:00000010:1.0:1788844879.702002:0:68594:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 00000000e990a3fd. 00000400:00000010:1.0:1788844879.702006:0:68594:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 000000003f3c6d43. 00000400:00000010:1.0:1788844879.702008:0:68594:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 00000000d69ff3ea. 00000400:00000010:1.0:1788844879.702010:0:68594:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 0000000027f265b9. 00000400:00000010:1.0:1788844879.702014:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000008015098. 00000400:00000010:1.0:1788844879.702027:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000bb78fac6. 00000400:00000010:1.0:1788844879.702034:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000d5dded7f. 00000400:00000010:1.0:1788844879.702036:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000a4ae4cda. 00000400:00000010:1.0:1788844879.702053:0:68594:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000009e812ea7. 00000400:00000200:1.0:1788844879.702059:0:68594:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004b62fb26 00000400:00000010:1.0:1788844879.702062:0:68594:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000004b62fb26. 00000400:00000200:1.0:1788844879.702067:0:68594:0:(api-ni.c:2393:lnet_push_target_event_handler()) type 6 status 0 unlinked 1 00000400:00000010:1.0:1788844879.702095:0:68594:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000ac4db73a. 00000400:00000400:1.0:1788844879.702102:0:68594:0:(lib-ptl.c:964:lnet_clear_lazy_portal()) Active lazy portal 0 on exit 00000400:00000010:1.0:1788844879.711577:0:68594:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000cb47952f. 00000400:00000200:1.0:1788844879.711594:0:68594:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a90fb858 00000400:00000010:1.0:1788844879.711597:0:68594:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000a90fb858. 00000400:00000010:2.0:1788844879.712014:0:68594:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 73 at 000000004c8a6973. 00000400:00000200:1.0:1788844879.712637:0:68232:0:(acceptor.c:659:lnet_acceptor()) Acceptor stopping 00000400:00000200:2.0:1788844879.712810:0:68594:0:(router.c:792:lnet_del_route()) Del route: net <255:65535> : gw 00000400:00000200:2.0:1788844879.712838:0:68594:0:(lo.c:49:lolnd_shutdown()) shutdown 00000400:00000010:2.0:1788844879.712842:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000aa794c31. 00000400:00000010:2.0:1788844879.712846:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000007457c726. 00000400:00000010:2.0:1788844879.712849:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000cb3d0ee1. 00000400:00000010:2.0:1788844879.712851:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000a79b3e1b. 00000400:00000010:2.0:1788844879.713205:0:68594:0:(config.c:293:lnet_ni_free()) kfreed 'ni': 560 at 00000000d53387a7. 00000400:00000010:2.0:1788844879.713219:0:68594:0:(config.c:329:lnet_net_free()) kfreed 'net': 176 at 000000001bd7867a. 00000800:00000200:2.1:1788844879.713239:0:68594:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 2, ksnr_connected a ksnr_max_conns 1 00000800:00000200:2.1:1788844879.713255:0:68594:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 3, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:2.1:1788844879.713257:0:68594:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:2.1:1788844879.713259:0:68594:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 1, ksnr_connected 0 ksnr_max_conns 1 00000800:00000010:2.1:1788844879.713263:0:68594:0:(socklnd.c:126:ksocknal_destroy_conn_cb()) kfreed 'conn_cb': 192 at 00000000ca3c6de5. 00000400:00020000:2.0:1788844879.713269:0:68594:0:(acceptor.c:252:lnet_acceptor_remove_socket()) Interface ens2 not found 00000800:00000200:1.0:1788844879.714230:0:68228:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 000000005636bcd6 00000800:00000010:1.0:1788844879.714237:0:68228:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 000000005636bcd6. 00000800:00000200:1.0:1788844879.714241:0:68228:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 000000004d481902 00000800:00000010:1.0:1788844879.714243:0:68228:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 000000004d481902. 00000800:00000200:1.0:1788844879.714374:0:68228:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 0000000019bdd1d9 00000800:00000010:1.0:1788844879.714376:0:68228:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 0000000019bdd1d9. 00000800:00000200:1.0:1788844879.714380:0:68228:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 0000000003ba3b00 00000800:00000200:1.0:1788844879.714388:0:68228:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.202.29@tcp 0000000065d6e929 deleted 00000800:00000010:1.0:1788844879.714391:0:68228:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 0000000065d6e929. 00000800:00000010:1.0:1788844879.714393:0:68228:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 0000000003ba3b00. 00000800:00000010:3.0:1788844879.726655:0:68594:0:(socklnd.c:2498:ksocknal_shutdown()) kfreed 'net': 208 at 00000000306fec4d. 00000400:02000000:3.0:1788844879.726681:0:68594:0:(api-ni.c:2563:lnet_clear_zombies_nis_locked()) Removed LNI 192.168.202.29@tcp 00000400:00000010:3.0:1788844879.732244:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000bc61d396. 00000400:00000010:3.0:1788844879.732267:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000008051bb76. 00000400:00000010:3.0:1788844879.732272:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000040cdf5ec. 00000400:00000010:3.0:1788844879.732274:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002210c718. 00000400:00000010:3.0:1788844879.732277:0:68594:0:(config.c:286:lnet_ni_free()) kfreed 'ni->ni_interface': 5 at 0000000012e9100f. 00000400:00000010:3.0:1788844879.732280:0:68594:0:(config.c:293:lnet_ni_free()) kfreed 'ni': 560 at 00000000faab73a8. 00000400:00000200:3.0:1788844879.732302:0:68594:0:(peer.c:620:lnet_peer_ni_clr_local_net_locked()) 00000000acd8e4cb nid 192.168.202.29@tcp detached from local net 00000400:00000010:3.0:1788844879.732332:0:68594:0:(config.c:329:lnet_net_free()) kfreed 'net': 176 at 000000006fc81c16. 00000400:00020000:3.0:1788844879.732372:0:68594:0:(acceptor.c:252:lnet_acceptor_remove_socket()) Interface ens2 not found 00000800:00000010:3.0:1788844879.732376:0:68594:0:(socklnd.c:2498:ksocknal_shutdown()) kfreed 'net': 208 at 0000000086a58983. 00000800:00000010:3.0:1788844879.732381:0:68594:0:(socklnd.c:2235:ksocknal_base_shutdown()) before NAL cleanup: kmem 418797 00000800:00000200:3.0:1788844879.732392:0:68594:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_GOING_DOWN, iface=lo ifindex 1 state 0 ns 000000004dc0f701 00000800:00000200:3.0:1788844879.732396:0:68594:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_DOWN, iface=lo ifindex 1 state 0 ns 000000004dc0f701 00000800:00000200:3.0:1788844879.732399:0:68594:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UNREGISTER, iface=lo ifindex 1 state 0 ns 000000004dc0f701 00000800:00000200:3.0:1788844879.732401:0:68594:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_GOING_DOWN, iface=ens2 ifindex 2 state 6 ns 000000004dc0f701 00000800:00000200:3.0:1788844879.732404:0:68594:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_DOWN, iface=ens2 ifindex 2 state 6 ns 000000004dc0f701 00000800:00000200:3.0:1788844879.732406:0:68594:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UNREGISTER, iface=ens2 ifindex 2 state 6 ns 000000004dc0f701 00000400:00000010:2.0:1788844879.747751:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 0000000099036bee. 00000400:00000010:2.0:1788844879.747814:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002f9f85de. 00000800:00000010:2.0:1788844879.747820:0:68594:0:(socklnd.c:2296:ksocknal_base_shutdown()) after NAL cleanup: kmem 418645 00000400:02000000:2.0:1788844879.747833:0:68594:0:(api-ni.c:2563:lnet_clear_zombies_nis_locked()) Removed LNI 192.168.202.29@tcp1 00000400:00000010:2.0:1788844879.750185:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000008334e28c. 00000400:00000010:2.0:1788844879.750198:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000006a949bd. 00000400:00000010:2.0:1788844879.750200:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000bc812c2a. 00000400:00000010:2.0:1788844879.750201:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000099b9809d. 00000400:00000010:2.0:1788844879.750204:0:68594:0:(config.c:286:lnet_ni_free()) kfreed 'ni->ni_interface': 5 at 00000000655fc3c2. 00000400:00000010:2.0:1788844879.750207:0:68594:0:(config.c:293:lnet_ni_free()) kfreed 'ni': 560 at 00000000cb247215. 00000400:00000200:2.0:1788844879.750225:0:68594:0:(peer.c:442:lnet_peer_del_locked()) peer 192.168.202.29@tcp1 00000400:00000200:2.0:1788844879.750232:0:68594:0:(peer.c:364:lnet_peer_detach_peer_ni_locked()) peer 192.168.202.29@tcp1 NID 192.168.202.29@tcp 00000400:00000200:2.0:1788844879.750235:0:68594:0:(peer.c:2392:lnet_destroy_peer_ni_locked()) 00000000acd8e4cb nid 192.168.202.29@tcp 00000400:00000010:2.0:1788844879.750238:0:68594:0:(peer.c:2422:lnet_destroy_peer_ni_locked()) kfreed 'lpni': 480 at 00000000acd8e4cb. 00000400:00000200:2.0:1788844879.750242:0:68594:0:(peer.c:213:lnet_destroy_peer_net_locked()) 00000000bc34c189 net tcp 00000400:00000010:2.0:1788844879.750244:0:68594:0:(peer.c:220:lnet_destroy_peer_net_locked()) kfreed 'lpn': 72 at 00000000bc34c189. 00000400:00000200:2.0:1788844879.750249:0:68594:0:(peer.c:364:lnet_peer_detach_peer_ni_locked()) peer 192.168.202.29@tcp1 NID 192.168.202.29@tcp1 00000400:00000200:2.0:1788844879.750252:0:68594:0:(peer.c:2392:lnet_destroy_peer_ni_locked()) 000000000a51eb25 nid 192.168.202.29@tcp1 00000400:00000010:2.0:1788844879.750254:0:68594:0:(peer.c:2422:lnet_destroy_peer_ni_locked()) kfreed 'lpni': 480 at 000000000a51eb25. 00000400:00000200:2.0:1788844879.750257:0:68594:0:(peer.c:213:lnet_destroy_peer_net_locked()) 00000000fed799a0 net tcp1 00000400:00000010:2.0:1788844879.750259:0:68594:0:(peer.c:220:lnet_destroy_peer_net_locked()) kfreed 'lpn': 72 at 00000000fed799a0. 00000400:00000200:2.0:1788844879.750263:0:68594:0:(peer.c:274:lnet_destroy_peer_locked()) 0000000031654556 nid 192.168.202.29@tcp1 00000400:00000010:2.0:1788844879.750266:0:68594:0:(peer.c:304:lnet_destroy_peer_locked()) kfreed 'lp': 360 at 0000000031654556. 00000400:00000010:2.0:1788844879.750272:0:68594:0:(config.c:329:lnet_net_free()) kfreed 'net': 176 at 00000000f25748dd. 00000400:00000010:2.0:1788844879.750280:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000070a783f. 00000400:00000010:2.0:1788844879.750283:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000005f6fd0eb. 00000400:00000010:2.0:1788844879.750290:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000078e45bfe. 00000400:00000010:2.0:1788844879.750293:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000aacf4d98. 00000400:00000010:2.0:1788844879.750296:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000000ba5a3e3. 00000400:00000010:2.0:1788844879.750298:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000009822bda0. 00000400:00000010:2.0:1788844879.750302:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000073a778bd. 00000400:00000010:2.0:1788844879.750305:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000006eef58f3. 00000400:00000010:2.0:1788844879.750306:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000022882eba. 00000400:00000010:2.0:1788844879.750307:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000008d92d1d0. 00000400:00000010:2.0:1788844879.750310:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000008608d7f3. 00000400:00000010:2.0:1788844879.750331:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000004a12cee2. 00000400:00000010:2.0:1788844879.750332:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000e622a591. 00000400:00000010:2.0:1788844879.750333:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000509589b9. 00000400:00000010:2.0:1788844879.750336:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000046f7e0d3. 00000400:00000010:2.0:1788844879.750339:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000a985dd30. 00000400:00000010:2.0:1788844879.750340:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000001fb966d6. 00000400:00000010:2.0:1788844879.750340:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000033331e74. 00000400:00000010:2.0:1788844879.750343:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000e54e880c. 00000400:00000010:2.0:1788844879.750345:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000af60611f. 00000400:00000010:2.0:1788844879.750346:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000ac38bab1. 00000400:00000010:2.0:1788844879.750346:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000006a673634. 00000400:00000010:2.0:1788844879.750349:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000004e1cda6e. 00000400:00000010:2.0:1788844879.750351:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000004c11046f. 00000400:00000010:2.0:1788844879.750352:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000003dcaf6c3. 00000400:00000010:2.0:1788844879.750353:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000040e59d94. 00000400:00000010:2.0:1788844879.750357:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000e0fb5a6a. 00000400:00000010:2.0:1788844879.750359:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000794d30fb. 00000400:00000010:2.0:1788844879.750360:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000083817372. 00000400:00000010:2.0:1788844879.750362:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000082aaed87. 00000400:00000010:2.0:1788844879.750366:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000006051c759. 00000400:00000010:2.0:1788844879.750368:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000008aed253d. 00000400:00000010:2.0:1788844879.750370:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000e96c2101. 00000400:00000010:2.0:1788844879.750371:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000006ac195db. 00000400:00000010:2.0:1788844879.750375:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000003c7946d5. 00000400:00000010:2.0:1788844879.750378:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000487f3d81. 00000400:00000010:2.0:1788844879.750379:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000673ec8c3. 00000400:00000010:2.0:1788844879.750381:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000be812680. 00000400:00000010:2.0:1788844879.750385:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000049f837f7. 00000400:00000010:2.0:1788844879.750387:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000746ba041. 00000400:00000010:2.0:1788844879.750388:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000006f91e6c2. 00000400:00000010:2.0:1788844879.750390:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000be64e6df. 00000400:00000010:2.0:1788844879.750394:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000004c570512. 00000400:00000010:2.0:1788844879.750396:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000d2188600. 00000400:00000010:2.0:1788844879.750403:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002de19088. 00000400:00000010:2.0:1788844879.750405:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000533aebf8. 00000400:00000010:2.0:1788844879.750409:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000beeb68e0. 00000400:00000010:2.0:1788844879.750413:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000004c7598be. 00000400:00000010:2.0:1788844879.750415:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000046f2779d. 00000400:00000010:2.0:1788844879.750416:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000fbcb27d0. 00000400:00000010:2.0:1788844879.750420:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000048876783. 00000400:00000010:2.0:1788844879.750422:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f858c11c. 00000400:00000010:2.0:1788844879.750424:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000b3ba702c. 00000400:00000010:2.0:1788844879.750426:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000636096a3. 00000400:00000010:2.0:1788844879.750430:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000aa6f72a4. 00000400:00000010:2.0:1788844879.750432:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f42e59ce. 00000400:00000010:2.0:1788844879.750433:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000cb33196d. 00000400:00000010:2.0:1788844879.750435:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000003f0b1dcc. 00000400:00000010:2.0:1788844879.750439:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000071fab5be. 00000400:00000010:2.0:1788844879.750442:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000e5bf5a26. 00000400:00000010:2.0:1788844879.750443:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000ed74736f. 00000400:00000010:2.0:1788844879.750445:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000be60938d. 00000400:00000010:2.0:1788844879.750449:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000052a76679. 00000400:00000010:2.0:1788844879.750452:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000004f6fe3dd. 00000400:00000010:2.0:1788844879.750453:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000092d5e100. 00000400:00000010:2.0:1788844879.750454:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000007712bc2f. 00000400:00000010:2.0:1788844879.750458:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000002383e933. 00000400:00000010:2.0:1788844879.750459:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000bde7bc46. 00000400:00000010:2.0:1788844879.750461:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000003383df86. 00000400:00000010:2.0:1788844879.750462:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000410237bb. 00000400:00000010:2.0:1788844879.750465:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000004c3f562e. 00000400:00000010:2.0:1788844879.750466:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000fca55343. 00000400:00000010:2.0:1788844879.750467:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000003e4ffdfb. 00000400:00000010:2.0:1788844879.750468:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000f05f8179. 00000400:00000010:2.0:1788844879.750470:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f8fd273d. 00000400:00000010:2.0:1788844879.750472:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000071003de9. 00000400:00000010:2.0:1788844879.750473:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000008325b807. 00000400:00000010:2.0:1788844879.750478:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000003e97ebac. 00000400:00000010:2.0:1788844879.750481:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000732bddc8. 00000400:00000010:2.0:1788844879.750483:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000c8960bc1. 00000400:00000010:2.0:1788844879.750484:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000c5c66057. 00000400:00000010:2.0:1788844879.750485:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000d4e6dfac. 00000400:00000010:2.0:1788844879.750488:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000000b186f96. 00000400:00000010:2.0:1788844879.750490:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000a8e6c637. 00000400:00000010:2.0:1788844879.750492:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000001c0f80ec. 00000400:00000010:2.0:1788844879.750492:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e6f3a77a. 00000400:00000010:2.0:1788844879.750495:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000fcd20a28. 00000400:00000010:2.0:1788844879.750496:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000086157974. 00000400:00000010:2.0:1788844879.750497:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000003307edf7. 00000400:00000010:2.0:1788844879.750498:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000000d8ff21a. 00000400:00000010:2.0:1788844879.750500:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f830d99c. 00000400:00000010:2.0:1788844879.750502:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000c6360905. 00000400:00000010:2.0:1788844879.750503:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000003c416875. 00000400:00000010:2.0:1788844879.750504:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000060c3a218. 00000400:00000010:2.0:1788844879.750507:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000058d5f192. 00000400:00000010:2.0:1788844879.750509:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000049eb34aa. 00000400:00000010:2.0:1788844879.750509:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000051b31e9e. 00000400:00000010:2.0:1788844879.750510:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000002b4ef567. 00000400:00000010:2.0:1788844879.750513:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000004b3231fa. 00000400:00000010:2.0:1788844879.750514:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000008e3f0f17. 00000400:00000010:2.0:1788844879.750515:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000d53e8531. 00000400:00000010:2.0:1788844879.750516:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000c29af030. 00000400:00000010:2.0:1788844879.750518:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000c587f8d0. 00000400:00000010:2.0:1788844879.750520:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000dfd23c39. 00000400:00000010:2.0:1788844879.750521:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000c3ca6ac0. 00000400:00000010:2.0:1788844879.750521:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000000e3ed993. 00000400:00000010:2.0:1788844879.750524:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000035c7b992. 00000400:00000010:2.0:1788844879.750526:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f8e8bdff. 00000400:00000010:2.0:1788844879.750527:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000fe98c810. 00000400:00000010:2.0:1788844879.750528:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e4078625. 00000400:00000010:2.0:1788844879.750534:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000005aefee44. 00000400:00000010:2.0:1788844879.750536:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000066542ff8. 00000400:00000010:2.0:1788844879.750536:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000082722c85. 00000400:00000010:2.0:1788844879.750537:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000001c2ac2af. 00000400:00000010:2.0:1788844879.750540:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000012bb5fad. 00000400:00000010:2.0:1788844879.750542:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000082b9c69. 00000400:00000010:2.0:1788844879.750543:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000ad2ebd98. 00000400:00000010:2.0:1788844879.750544:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000bad21083. 00000400:00000010:2.0:1788844879.750546:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000b7b45c03. 00000400:00000010:2.0:1788844879.750547:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000207bc300. 00000400:00000010:2.0:1788844879.750548:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000dc923f14. 00000400:00000010:2.0:1788844879.750549:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000039a56ea3. 00000400:00000010:2.0:1788844879.750552:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000025943f65. 00000400:00000010:2.0:1788844879.750553:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000fe5276fb. 00000400:00000010:2.0:1788844879.750554:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000550e30e3. 00000400:00000010:2.0:1788844879.750555:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000bf7e8b72. 00000400:00000010:2.0:1788844879.750558:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000007e760ea. 00000400:00000010:2.0:1788844879.750560:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000ba5dbec0. 00000400:00000010:2.0:1788844879.750560:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000d8c62805. 00000400:00000010:2.0:1788844879.750561:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000006afbcb62. 00000400:00000010:2.0:1788844879.750565:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000535203ad. 00000400:00000010:2.0:1788844879.750566:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000d4109edb. 00000400:00000010:2.0:1788844879.750567:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000001e0fde64. 00000400:00000010:2.0:1788844879.750568:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000015374619. 00000400:00000010:2.0:1788844879.750572:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000099676174. 00000400:00000010:2.0:1788844879.750575:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000e89178ad. 00000400:00000010:2.0:1788844879.750576:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000008ce46f14. 00000400:00000010:2.0:1788844879.750578:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000ca425fd7. 00000400:00000010:2.0:1788844879.750582:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000005d51b27f. 00000400:00000010:2.0:1788844879.750584:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000002d5f1d21. 00000400:00000010:2.0:1788844879.750586:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000a6b8db6d. 00000400:00000010:2.0:1788844879.750588:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000d2b64378. 00000400:00000010:2.0:1788844879.750591:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000148b82bf. 00000400:00000010:2.0:1788844879.750599:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000008ddde762. 00000400:00000010:2.0:1788844879.750601:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000d19d5c46. 00000400:00000010:2.0:1788844879.750603:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000f5d7249b. 00000400:00000010:2.0:1788844879.750606:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ce23c6c8. 00000400:00000010:2.0:1788844879.750609:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000afcd23c6. 00000400:00000010:2.0:1788844879.750610:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000007c655c6c. 00000400:00000010:2.0:1788844879.750612:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000002e8d245c. 00000400:00000010:2.0:1788844879.750615:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000008f2d5864. 00000400:00000010:2.0:1788844879.750618:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000008c904632. 00000400:00000010:2.0:1788844879.750619:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000694f14af. 00000400:00000010:2.0:1788844879.750621:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000584e00a3. 00000400:00000010:2.0:1788844879.750624:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000e492d794. 00000400:00000010:2.0:1788844879.750639:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000002b278a64. 00000400:00000010:2.0:1788844879.750640:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000032772fda. 00000400:00000010:2.0:1788844879.750641:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000000c9f347a. 00000400:00000010:2.0:1788844879.750644:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ea9620cd. 00000400:00000010:2.0:1788844879.750646:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000003d4a08f6. 00000400:00000010:2.0:1788844879.750647:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000003752fe09. 00000400:00000010:2.0:1788844879.750648:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000bba856b4. 00000400:00000010:2.0:1788844879.750650:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f4841a6c. 00000400:00000010:2.0:1788844879.750651:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000043d7f657. 00000400:00000010:2.0:1788844879.750652:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000095e0fb09. 00000400:00000010:2.0:1788844879.750653:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000008ce8d092. 00000400:00000010:2.0:1788844879.750656:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000e29cd06f. 00000400:00000010:2.0:1788844879.750657:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000001ba0346c. 00000400:00000010:2.0:1788844879.750658:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000a2513ec0. 00000400:00000010:2.0:1788844879.750659:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000004df8bfca. 00000400:00000010:2.0:1788844879.750662:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000094e9313b. 00000400:00000010:2.0:1788844879.750663:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000fbb73f0c. 00000400:00000010:2.0:1788844879.750664:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000814a01ff. 00000400:00000010:2.0:1788844879.750665:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000000d0584b1. 00000400:00000010:2.0:1788844879.750668:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000090d28e3a. 00000400:00000010:2.0:1788844879.750670:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000001a09bc38. 00000400:00000010:2.0:1788844879.750674:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000000868a3d0. 00000400:00000010:2.0:1788844879.750676:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000090c9d724. 00000400:00000010:2.0:1788844879.750678:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000b6393d4e. 00000400:00000010:2.0:1788844879.750680:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000003c128bf3. 00000400:00000010:2.0:1788844879.750681:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000006ede8a16. 00000400:00000010:2.0:1788844879.750682:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000439803c5. 00000400:00000010:2.0:1788844879.750684:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000004d403923. 00000400:00000010:2.0:1788844879.750685:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000b8c5ba7f. 00000400:00000010:2.0:1788844879.750686:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000d61be514. 00000400:00000010:2.0:1788844879.750687:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000f62836a0. 00000400:00000010:2.0:1788844879.750692:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000cce5a98d. 00000400:00000010:2.0:1788844879.750694:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000074d2f83e. 00000400:00000010:2.0:1788844879.750696:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000d4c04789. 00000400:00000010:2.0:1788844879.750698:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000a66f45f9. 00000400:00000010:2.0:1788844879.750701:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000b06228b2. 00000400:00000010:2.0:1788844879.750703:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000002accb2c7. 00000400:00000010:2.0:1788844879.750705:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000000da25629. 00000400:00000010:2.0:1788844879.750706:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000aa548f2f. 00000400:00000010:2.0:1788844879.750711:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000011c2bc27. 00000400:00000010:2.0:1788844879.750713:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000004a544abe. 00000400:00000010:2.0:1788844879.750715:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000fe9f290d. 00000400:00000010:2.0:1788844879.750716:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000d34354f1. 00000400:00000010:2.0:1788844879.750721:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000001c0ab122. 00000400:00000010:2.0:1788844879.750723:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000ad341a92. 00000400:00000010:2.0:1788844879.750725:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000004d22b8e9. 00000400:00000010:2.0:1788844879.750728:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000ac62a264. 00000400:00000010:2.0:1788844879.750732:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000041fda1eb. 00000400:00000010:2.0:1788844879.750762:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000024ef0b92. 00000400:00000010:2.0:1788844879.750765:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000d6e54e6a. 00000400:00000010:2.0:1788844879.750771:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000006e877309. 00000400:00000010:2.0:1788844879.750775:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000091cba7af. 00000400:00000010:2.0:1788844879.750777:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000001e8ecbb5. 00000400:00000010:2.0:1788844879.750820:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000005e19d104. 00000400:00000010:2.0:1788844879.750828:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e4cb4805. 00000400:00000010:2.0:1788844879.750832:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000b075f539. 00000400:00000010:2.0:1788844879.750835:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000bc3994fc. 00000400:00000010:2.0:1788844879.750837:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000009d278723. 00000400:00000010:2.0:1788844879.750839:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000000df8487c. 00000400:00000010:2.0:1788844879.750842:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ac6fdf8f. 00000400:00000010:2.0:1788844879.750845:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000737da499. 00000400:00000010:2.0:1788844879.750846:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000003ba48064. 00000400:00000010:2.0:1788844879.750848:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000ba4b6ecb. 00000400:00000010:2.0:1788844879.750855:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000085d84fe0. 00000400:00000010:2.0:1788844879.750858:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000654bf4b2. 00000400:00000010:2.0:1788844879.750860:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000f2fedfc5. 00000400:00000010:2.0:1788844879.750861:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000a9e95ace. 00000400:00000010:2.0:1788844879.750865:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000666acba5. 00000400:00000010:2.0:1788844879.750867:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000042b64368. 00000400:00000010:2.0:1788844879.750869:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000d89ff0c9. 00000400:00000010:2.0:1788844879.750870:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000081355940. 00000400:00000010:2.0:1788844879.750874:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f2896859. 00000400:00000010:2.0:1788844879.750876:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000001b4c5c26. 00000400:00000010:2.0:1788844879.750878:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000060ec8392. 00000400:00000010:2.0:1788844879.750879:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000d55bb4a0. 00000400:00000010:2.0:1788844879.750883:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000001dc6f449. 00000400:00000010:2.0:1788844879.750887:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000002e537f08. 00000400:00000010:2.0:1788844879.750888:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000034e6b041. 00000400:00000010:2.0:1788844879.750890:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000d3a31ee7. 00000400:00000010:2.0:1788844879.750894:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ce632e44. 00000400:00000010:2.0:1788844879.750896:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000140c6969. 00000400:00000010:2.0:1788844879.750897:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000552b0ef8. 00000400:00000010:2.0:1788844879.750898:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000014b7cb64. 00000400:00000010:2.0:1788844879.750901:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000365db3e4. 00000400:00000010:2.0:1788844879.750903:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000009a9d7211. 00000400:00000010:2.0:1788844879.750904:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000de5831f8. 00000400:00000010:2.0:1788844879.750904:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000009829b257. 00000400:00000010:2.0:1788844879.750913:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000026a45d92. 00000400:00000010:2.0:1788844879.750915:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000b623ff4b. 00000400:00000010:2.0:1788844879.750917:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000072e0489. 00000400:00000010:2.0:1788844879.750919:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000734994e0. 00000400:00000010:2.0:1788844879.750922:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000dd0c6b59. 00000400:00000010:2.0:1788844879.750924:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000008b26e223. 00000400:00000010:2.0:1788844879.750926:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000d0417543. 00000400:00000010:2.0:1788844879.750927:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000d99c3fdc. 00000400:00000010:2.0:1788844879.750931:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000a5ec57bb. 00000400:00000010:2.0:1788844879.750934:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000058d09cc6. 00000400:00000010:2.0:1788844879.750936:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002bfd2633. 00000400:00000010:2.0:1788844879.750937:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000008135f41e. 00000400:00000010:2.0:1788844879.750941:0:68594:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000afe01e78. 00000400:00000010:2.0:1788844879.750943:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000a9e2ed54. 00000400:00000010:2.0:1788844879.750945:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000da489f6e. 00000400:00000010:2.0:1788844879.750946:0:68594:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000024385f93. 00000400:00000010:2.0:1788844879.750948:0:68594:0:(lib-ptl.c:854:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals': 512 at 00000000fa760abd. 00000400:00000010:2.0:1788844879.750952:0:68594:0:(api-ni.c:1205:lnet_res_container_cleanup()) kfreed 'rec->rec_lh_hash': 65536 at 0000000006e052d1. 00000400:00000010:2.0:1788844879.750969:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000be36d944. 00000400:00000010:2.0:1788844879.750970:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000f3c35bf7. 00000400:00000010:2.0:1788844879.750972:0:68594:0:(api-ni.c:1205:lnet_res_container_cleanup()) kfreed 'rec->rec_lh_hash': 65536 at 00000000b244745d. 00000400:00000010:2.0:1788844879.750974:0:68594:0:(lib-msg.c:1308:lnet_msg_container_cleanup()) kfreed 'container->msc_finalizers': 32 at 0000000020509cf5. 00000400:00000010:2.0:1788844879.750976:0:68594:0:(lib-msg.c:1314:lnet_msg_container_cleanup()) kfreed 'container->msc_resenders': 32 at 00000000231f6131. 00000400:00000010:2.0:1788844879.750978:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 00000000e85a91b5. 00000400:00000010:2.0:1788844879.750980:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000e275d772. 00000400:00000010:2.0:1788844879.750984:0:68594:0:(peer.c:85:lnet_peer_tables_destroy()) kfreed 'hash': 8192 at 000000001c3d3225. 00000400:00000010:2.0:1788844879.750992:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 000000008c32b593. 00000400:00000010:2.0:1788844879.750994:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000381127ab. 00000400:00000010:2.0:1788844879.750996:0:68594:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 00000000cadf9311. 00000400:00000010:2.0:1788844879.750997:0:68594:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000d72a03a9. 00000400:00000010:2.0:1788844879.750999:0:68594:0:(api-ni.c:700:lnet_destroy_remote_nets_table()) kfreed 'the_lnet.ln_remote_nets_hash': 1024 at 000000007b1a5f43. 00000001:00000200:2.0:1788844879.751003:0:68594:0:(udsp.c:1126:lnet_udsp_destroy()) Destroying UDSPs in the system 00000001:02000400:2.0:1788844879.773439:0:68595:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure 00000400:00000010:0.0:1788844879.798221:0:68596:0:(api-ni.c:676:lnet_create_remote_nets_table()) alloc '(hash)': 1024 at 000000000956d444. 00000400:00000010:0.0:1788844879.798237:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000044e89cc2. 00000400:00000010:0.0:1788844879.798243:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 000000007a2ca541. 00000400:00000010:0.0:1788844879.798247:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000c04d4d2a. 00000400:00000010:0.0:1788844879.798248:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 000000008e269c04. 00000400:00000010:0.0:1788844879.798255:0:68596:0:(peer.c:109:lnet_peer_tables_create()) alloc '(hash)': 8192 at 0000000003ba3b00. 00000400:00000010:0.0:1788844879.798261:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000c45e19d2. 00000400:00000010:0.0:1788844879.798263:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 000000000b454fab. 00000400:00000010:0.0:1788844879.798267:0:68596:0:(lib-msg.c:1338:lnet_msg_container_setup()) alloc '(container->msc_finalizers)': 32 at 00000000b7ef994e. 00000400:00000010:0.0:1788844879.798270:0:68596:0:(lib-msg.c:1348:lnet_msg_container_setup()) alloc '(container->msc_resenders)': 32 at 00000000d85ba17b. 00000400:00000010:0.0:1788844879.798354:0:68596:0:(api-ni.c:1227:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 000000005572738d. 00000400:00000010:0.0:1788844879.798373:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000004f388c65. 00000400:00000010:0.0:1788844879.798376:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001772a699. 00000400:00000010:0.0:1788844879.798447:0:68596:0:(api-ni.c:1227:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 00000000e3176fe9. 00000400:00000010:0.0:1788844879.798466:0:68596:0:(lib-ptl.c:864:lnet_portals_create()) alloc '(the_lnet.ln_portals)': 512 at 00000000aff1e4cf. 00000400:00000010:0.0:1788844879.798471:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000888a09bb. 00000400:00000010:0.0:1788844879.798473:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000008a3cf2bb. 00000400:00000010:0.0:1788844879.798476:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006312873b. 00000400:00000010:0.0:1788844879.798482:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000062b2d79e. 00000400:00000010:0.0:1788844879.798485:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000101b3909. 00000400:00000010:0.0:1788844879.798488:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000002691533. 00000400:00000010:0.0:1788844879.798490:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000059acbdd4. 00000400:00000010:0.0:1788844879.798494:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004950b76d. 00000400:00000010:0.0:1788844879.798497:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000002e9f5976. 00000400:00000010:0.0:1788844879.798498:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000493db505. 00000400:00000010:0.0:1788844879.798500:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005836113b. 00000400:00000010:0.0:1788844879.798505:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002ca18eb7. 00000400:00000010:0.0:1788844879.798508:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e99f0f74. 00000400:00000010:0.0:1788844879.798509:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000045514389. 00000400:00000010:0.0:1788844879.798511:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a7f7befd. 00000400:00000010:0.0:1788844879.798518:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d7072bc3. 00000400:00000010:0.0:1788844879.798520:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ad4dd87a. 00000400:00000010:0.0:1788844879.798522:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000baf33e5d. 00000400:00000010:0.0:1788844879.798524:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002b080a5b. 00000400:00000010:0.0:1788844879.798529:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d584d457. 00000400:00000010:0.0:1788844879.798531:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000028740d62. 00000400:00000010:0.0:1788844879.798532:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a9db2fb4. 00000400:00000010:0.0:1788844879.798534:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000039cdd270. 00000400:00000010:0.0:1788844879.798539:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000dc206ed9. 00000400:00000010:0.0:1788844879.798542:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e85f0291. 00000400:00000010:0.0:1788844879.798543:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000023af65e. 00000400:00000010:0.0:1788844879.798545:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000468d16ad. 00000400:00000010:0.0:1788844879.798550:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b555330b. 00000400:00000010:0.0:1788844879.798553:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000006e31b303. 00000400:00000010:0.0:1788844879.798555:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d24b9af7. 00000400:00000010:0.0:1788844879.798564:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d447cea9. 00000400:00000010:0.0:1788844879.798572:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004c570512. 00000400:00000010:0.0:1788844879.798575:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ac6e27cf. 00000400:00000010:0.0:1788844879.798577:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ba407268. 00000400:00000010:0.0:1788844879.798579:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001db122b3. 00000400:00000010:0.0:1788844879.798584:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000049f837f7. 00000400:00000010:0.0:1788844879.798586:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000062556f66. 00000400:00000010:0.0:1788844879.798588:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e240750e. 00000400:00000010:0.0:1788844879.798590:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003f207307. 00000400:00000010:0.0:1788844879.798595:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003c7946d5. 00000400:00000010:0.0:1788844879.798598:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000bc34c189. 00000400:00000010:0.0:1788844879.798600:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000c2f823b2. 00000400:00000010:0.0:1788844879.798602:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003ad6ea45. 00000400:00000010:0.0:1788844879.798607:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006051c759. 00000400:00000010:0.0:1788844879.798610:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a9800616. 00000400:00000010:0.0:1788844879.798612:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000028083766. 00000400:00000010:0.0:1788844879.798614:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000305212d9. 00000400:00000010:0.0:1788844879.798619:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e0fb5a6a. 00000400:00000010:0.0:1788844879.798622:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000026ed2ce0. 00000400:00000010:0.0:1788844879.798623:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000eed24453. 00000400:00000010:0.0:1788844879.798625:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000072be2e75. 00000400:00000010:0.0:1788844879.798629:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004e1cda6e. 00000400:00000010:0.0:1788844879.798632:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000002b712343. 00000400:00000010:0.0:1788844879.798633:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000004b77f11c. 00000400:00000010:0.0:1788844879.798635:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ee49a149. 00000400:00000010:0.0:1788844879.798640:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e54e880c. 00000400:00000010:0.0:1788844879.798642:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000581ae48b. 00000400:00000010:0.0:1788844879.798644:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000363a54ae. 00000400:00000010:0.0:1788844879.798645:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001d9bb8c8. 00000400:00000010:0.0:1788844879.798650:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000046f7e0d3. 00000400:00000010:0.0:1788844879.798653:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000200e3a59. 00000400:00000010:0.0:1788844879.798655:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005f6e227b. 00000400:00000010:0.0:1788844879.798662:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000e5d5b49. 00000400:00000010:0.0:1788844879.798668:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f830d99c. 00000400:00000010:0.0:1788844879.798671:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000835f7a68. 00000400:00000010:0.0:1788844879.798673:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000004f96d58c. 00000400:00000010:0.0:1788844879.798675:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000007e0d620. 00000400:00000010:0.0:1788844879.798681:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000fcd20a28. 00000400:00000010:0.0:1788844879.798683:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a9c9fd57. 00000400:00000010:0.0:1788844879.798685:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000006591c073. 00000400:00000010:0.0:1788844879.798687:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000040cdf5ec. 00000400:00000010:0.0:1788844879.798692:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000000b186f96. 00000400:00000010:0.0:1788844879.798695:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000006764a14e. 00000400:00000010:0.0:1788844879.798697:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000098462bc. 00000400:00000010:0.0:1788844879.798699:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000bc61d396. 00000400:00000010:0.0:1788844879.798704:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000732bddc8. 00000400:00000010:0.0:1788844879.798706:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000060e4fd94. 00000400:00000010:0.0:1788844879.798708:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000b308ad10. 00000400:00000010:0.0:1788844879.798710:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009b68a820. 00000400:00000010:0.0:1788844879.798716:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f8fd273d. 00000400:00000010:0.0:1788844879.798718:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000072f5f159. 00000400:00000010:0.0:1788844879.798720:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000cf697eee. 00000400:00000010:0.0:1788844879.798722:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006898b918. 00000400:00000010:0.0:1788844879.798727:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004c3f562e. 00000400:00000010:0.0:1788844879.798729:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000f9964600. 00000400:00000010:0.0:1788844879.798731:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000741f4313. 00000400:00000010:0.0:1788844879.798733:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009a63222b. 00000400:00000010:0.0:1788844879.798749:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002383e933. 00000400:00000010:0.0:1788844879.798752:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000007966fc70. 00000400:00000010:0.0:1788844879.798754:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000f355510d. 00000400:00000010:0.0:1788844879.798756:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000091848fd4. 00000400:00000010:0.0:1788844879.798762:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000052a76679. 00000400:00000010:0.0:1788844879.798765:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000009b36ed2c. 00000400:00000010:0.0:1788844879.798766:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ee8dfa2e. 00000400:00000010:0.0:1788844879.798773:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000018f4d00d. 00000400:00000010:0.0:1788844879.798807:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000071fab5be. 00000400:00000010:0.0:1788844879.798812:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000b8a50ccf. 00000400:00000010:0.0:1788844879.798814:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000009116a73c. 00000400:00000010:0.0:1788844879.798816:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ef1655be. 00000400:00000010:0.0:1788844879.798821:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000aa6f72a4. 00000400:00000010:0.0:1788844879.798824:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000000a6d6962. 00000400:00000010:0.0:1788844879.798826:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000fb2d6842. 00000400:00000010:0.0:1788844879.798828:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000019f8f883. 00000400:00000010:0.0:1788844879.798832:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000048876783. 00000400:00000010:0.0:1788844879.798835:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000003bc85b5d. 00000400:00000010:0.0:1788844879.798837:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000161d5ed6. 00000400:00000010:0.0:1788844879.798839:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005a7f0f8f. 00000400:00000010:0.0:1788844879.798844:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000beeb68e0. 00000400:00000010:0.0:1788844879.798847:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000092971a5c. 00000400:00000010:0.0:1788844879.798850:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000072acab06. 00000400:00000010:0.0:1788844879.798851:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c4c8bad6. 00000400:00000010:0.0:1788844879.798858:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005aefee44. 00000400:00000010:0.0:1788844879.798861:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000062fe75ae. 00000400:00000010:0.0:1788844879.798863:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000003c313220. 00000400:00000010:0.0:1788844879.798865:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f66e8904. 00000400:00000010:0.0:1788844879.798870:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000035c7b992. 00000400:00000010:0.0:1788844879.798874:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000814c4bb6. 00000400:00000010:0.0:1788844879.798876:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000007060eeb1. 00000400:00000010:0.0:1788844879.798877:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003a5177eb. 00000400:00000010:0.0:1788844879.798883:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000c587f8d0. 00000400:00000010:0.0:1788844879.798886:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000005a95a6de. 00000400:00000010:0.0:1788844879.798888:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000007d1d0893. 00000400:00000010:0.0:1788844879.798890:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c64cc704. 00000400:00000010:0.0:1788844879.798894:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004b3231fa. 00000400:00000010:0.0:1788844879.798897:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000dc729434. 00000400:00000010:0.0:1788844879.798898:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002577d682. 00000400:00000010:0.0:1788844879.798906:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000009b2f5c5. 00000400:00000010:0.0:1788844879.798911:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000058d5f192. 00000400:00000010:0.0:1788844879.798914:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000006c09f49. 00000400:00000010:0.0:1788844879.798916:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000feebb32e. 00000400:00000010:0.0:1788844879.798917:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a30ddda9. 00000400:00000010:0.0:1788844879.798924:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b7b45c03. 00000400:00000010:0.0:1788844879.798927:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a39c78b7. 00000400:00000010:0.0:1788844879.798928:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000012f5b268. 00000400:00000010:0.0:1788844879.798930:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b83ed0df. 00000400:00000010:0.0:1788844879.798936:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000012bb5fad. 00000400:00000010:0.0:1788844879.798938:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000007178c36a. 00000400:00000010:0.0:1788844879.798940:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002f9882bf. 00000400:00000010:0.0:1788844879.798942:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002c777628. 00000400:00000010:0.0:1788844879.798959:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008608d7f3. 00000400:00000010:0.0:1788844879.798963:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000f93ac11e. 00000400:00000010:0.0:1788844879.798965:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000044352509. 00000400:00000010:0.0:1788844879.798968:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fc2d389e. 00000400:00000010:0.0:1788844879.798972:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001c3d3225. 00000400:00000010:0.0:1788844879.798974:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000db66b092. 00000400:00000010:0.0:1788844879.798976:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000014991cc7. 00000400:00000010:0.0:1788844879.798978:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000cbe9af0b. 00000400:00000010:0.0:1788844879.798982:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000078e45bfe. 00000400:00000010:0.0:1788844879.798985:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000014da4132. 00000400:00000010:0.0:1788844879.798987:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000bb786270. 00000400:00000010:0.0:1788844879.798989:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000503c1090. 00000400:00000010:0.0:1788844879.798992:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000073a778bd. 00000400:00000010:0.0:1788844879.798995:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000235025b8. 00000400:00000010:0.0:1788844879.798996:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002551acd8. 00000400:00000010:0.0:1788844879.798998:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005c24ceee. 00000400:00000010:0.0:1788844879.799009:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001c0ab122. 00000400:00000010:0.0:1788844879.799013:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000c7581a2f. 00000400:00000010:0.0:1788844879.799015:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ab70581a. 00000400:00000010:0.0:1788844879.799022:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000ee4022c. 00000400:00000010:0.0:1788844879.799026:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000cce5a98d. 00000400:00000010:0.0:1788844879.799029:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000bfd6027d. 00000400:00000010:0.0:1788844879.799031:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e964c8d9. 00000400:00000010:0.0:1788844879.799033:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000029b104f2. 00000400:00000010:0.0:1788844879.799038:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b06228b2. 00000400:00000010:0.0:1788844879.799040:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000006fa59e70. 00000400:00000010:0.0:1788844879.799042:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000000f30b4f1. 00000400:00000010:0.0:1788844879.799044:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000322b02b6. 00000400:00000010:0.0:1788844879.799049:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000011c2bc27. 00000400:00000010:0.0:1788844879.799052:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000d8145822. 00000400:00000010:0.0:1788844879.799054:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000000140882c. 00000400:00000010:0.0:1788844879.799056:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003efbdae6. 00000400:00000010:0.0:1788844879.799066:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000090d28e3a. 00000400:00000010:0.0:1788844879.799070:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a9becf6e. 00000400:00000010:0.0:1788844879.799072:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000091e2ad38. 00000400:00000010:0.0:1788844879.799074:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000060204fe1. 00000400:00000010:0.0:1788844879.799077:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b6393d4e. 00000400:00000010:0.0:1788844879.799079:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000979c21a9. 00000400:00000010:0.0:1788844879.799081:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000080d46809. 00000400:00000010:0.0:1788844879.799083:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005ac03011. 00000400:00000010:0.0:1788844879.799086:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004d403923. 00000400:00000010:0.0:1788844879.799089:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000007dbb7b21. 00000400:00000010:0.0:1788844879.799091:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000050551255. 00000400:00000010:0.0:1788844879.799093:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000082d1b873. 00000400:00000010:0.0:1788844879.799098:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000094e9313b. 00000400:00000010:0.0:1788844879.799100:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000353dad28. 00000400:00000010:0.0:1788844879.799102:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000004bc073f9. 00000400:00000010:0.0:1788844879.799103:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007c668254. 00000400:00000010:0.0:1788844879.799113:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e492d794. 00000400:00000010:0.0:1788844879.799117:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000001b0e9840. 00000400:00000010:0.0:1788844879.799118:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000009d92ba7b. 00000400:00000010:0.0:1788844879.799124:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005c7b5d24. 00000400:00000010:0.0:1788844879.799128:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ea9620cd. 00000400:00000010:0.0:1788844879.799130:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000024a8ae36. 00000400:00000010:0.0:1788844879.799132:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002226e38c. 00000400:00000010:0.0:1788844879.799133:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e9443202. 00000400:00000010:0.0:1788844879.799136:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f4841a6c. 00000400:00000010:0.0:1788844879.799139:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a711f7ba. 00000400:00000010:0.0:1788844879.799140:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000fcbb744a. 00000400:00000010:0.0:1788844879.799142:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002631df0d. 00000400:00000010:0.0:1788844879.799145:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e29cd06f. 00000400:00000010:0.0:1788844879.799147:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000076460c26. 00000400:00000010:0.0:1788844879.799149:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000083786cbf. 00000400:00000010:0.0:1788844879.799150:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000621c622. 00000400:00000010:0.0:1788844879.799161:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ce23c6c8. 00000400:00000010:0.0:1788844879.799164:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000004500d76a. 00000400:00000010:0.0:1788844879.799166:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000dde3929c. 00000400:00000010:0.0:1788844879.799168:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000de099ecc. 00000400:00000010:0.0:1788844879.799172:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008f2d5864. 00000400:00000010:0.0:1788844879.799175:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000004f1bb23c. 00000400:00000010:0.0:1788844879.799176:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001681843c. 00000400:00000010:0.0:1788844879.799178:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000326701d8. 00000400:00000010:0.0:1788844879.799183:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005d51b27f. 00000400:00000010:0.0:1788844879.799185:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000b5291ce4. 00000400:00000010:0.0:1788844879.799187:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000162133db. 00000400:00000010:0.0:1788844879.799189:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003cc657d3. 00000400:00000010:0.0:1788844879.799193:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000148b82bf. 00000400:00000010:0.0:1788844879.799195:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000002e43d554. 00000400:00000010:0.0:1788844879.799197:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e329392f. 00000400:00000010:0.0:1788844879.799199:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000f199da0. 00000400:00000010:0.0:1788844879.799208:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000025943f65. 00000400:00000010:0.0:1788844879.799211:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000aa90fcc5. 00000400:00000010:0.0:1788844879.799213:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000072c72039. 00000400:00000010:0.0:1788844879.799218:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f60f4112. 00000400:00000010:0.0:1788844879.799222:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000007e760ea. 00000400:00000010:0.0:1788844879.799225:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000f53e5427. 00000400:00000010:0.0:1788844879.799226:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a60a864c. 00000400:00000010:0.0:1788844879.799228:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000099de7f65. 00000400:00000010:0.0:1788844879.799232:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000535203ad. 00000400:00000010:0.0:1788844879.799235:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000004ccc80bf. 00000400:00000010:0.0:1788844879.799236:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005c6b28d3. 00000400:00000010:0.0:1788844879.799238:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b33e6f99. 00000400:00000010:0.0:1788844879.799243:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000099676174. 00000400:00000010:0.0:1788844879.799245:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000019ef2462. 00000400:00000010:0.0:1788844879.799247:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a8e07715. 00000400:00000010:0.0:1788844879.799249:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000757bc96f. 00000400:00000010:0.0:1788844879.799259:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006b982f8a. 00000400:00000010:0.0:1788844879.799262:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000003aa18ec1. 00000400:00000010:0.0:1788844879.799264:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000da79684b. 00000400:00000010:0.0:1788844879.799266:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d3850584. 00000400:00000010:0.0:1788844879.799271:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000cf800bcd. 00000400:00000010:0.0:1788844879.799273:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000b8a26942. 00000400:00000010:0.0:1788844879.799274:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005822650b. 00000400:00000010:0.0:1788844879.799276:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000836770ac. 00000400:00000010:0.0:1788844879.799281:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005b44c76a. 00000400:00000010:0.0:1788844879.799283:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000d5f64747. 00000400:00000010:0.0:1788844879.799285:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000093d98ddc. 00000400:00000010:0.0:1788844879.799286:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000188fd018. 00000400:00000010:0.0:1788844879.799291:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000000c0d76c6. 00000400:00000010:0.0:1788844879.799294:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000c517f7e5. 00000400:00000010:0.0:1788844879.799295:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002f9f85de. 00000400:00000010:0.0:1788844879.799297:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000700da446. 00000400:00000010:0.0:1788844879.799307:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008feb9acd. 00000400:00000010:0.0:1788844879.799312:0:68596:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000016024adb. 00000400:00000010:0.0:1788844879.799314:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002210c718. 00000400:00000010:0.0:1788844879.799321:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fa5f2aa8. 00000400:00000010:0.0:1788844879.799327:0:68596:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000df507ca1. 00000400:00000010:0.0:1788844879.799330:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000008ccbb23c. 00000400:00000010:0.0:1788844879.799332:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a4a28bc5. 00000400:00000010:0.0:1788844879.799337:0:68596:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 00000000641dd1e2. 00000400:00000010:0.0:1788844879.799343:0:68596:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 560 at 000000005fc77b98. 00000400:00000010:0.0:1788844879.799345:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000dbcf86c9. 00000400:00000010:0.0:1788844879.799347:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000068c16029. 00000400:00000010:0.0:1788844879.799348:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a89d7f34. 00000400:00000010:0.0:1788844879.799350:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000070e563cb. 00000400:00000010:0.0:1788844879.799367:0:68596:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 41 at 00000000a1603e8c. 00000400:00000010:0.0:1788844879.799375:0:68596:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000cdd306f0. 00000400:00000010:0.0:1788844879.799390:0:68596:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000072130235. 00000400:00000200:0.0:1788844879.799402:0:68596:0:(lib-ptl.c:918:LNetSetLazyPortal()) Setting portal 0 lazy 00000400:00000010:0.0:1788844879.799406:0:68596:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000024ee44a7. 00000400:00000010:0.0:1788844879.799409:0:68596:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000074a32fc1. 00000400:00000010:0.0:1788844879.799412:0:68596:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000008812e9f4. 00000400:00000200:0.0:1788844879.799414:0:68596:0:(api-ni.c:2383:lnet_push_target_post()) posted push target 0000000024ee44a7 00000400:00000200:0.0:1788844879.799417:0:68596:0:(api-ni.c:2340:lnet_push_target_resize()) nbytes 272 success 00000400:00000010:0.0:1788844879.799420:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002780725a. 00000400:00000010:0.0:1788844879.799422:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d8d54585. 00000400:00000010:0.0:1788844879.799424:0:68596:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000f11e7c3e. 00000400:00000010:0.0:1788844879.799426:0:68596:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001eee6f64. 00000400:00000200:1.0:1788844879.801206:0:68596:0:(peer.c:4457:lnet_peer_discovery_start()) discovery start: 0 00000400:00000200:3.0:1788844879.803005:0:68598:0:(peer.c:4283:lnet_peer_discovery()) started 00000400:00000010:3.0:1788844879.839275:0:68599:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000bc565e63. 00000400:00000010:3.0:1788844879.839306:0:68599:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000bc565e63. 00000400:00000010:3.0:1788844879.840711:0:68599:0:(api-ni.c:7409:lnet_route_show_start()) alloc '(rlist)': 16 at 00000000d192559b. 00000400:00000010:3.0:1788844879.840749:0:68599:0:(api-ni.c:7309:lnet_route_show_done()) kfreed 'rlist': 16 at 00000000d192559b. 00000400:00000010:3.0:1788844879.842084:0:68599:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 236 at 0000000086a58983. 00000400:00000010:3.0:1788844879.842100:0:68599:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 236 at 0000000086a58983. 00000400:00000010:3.0:1788844879.842341:0:68599:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 00000000d192559b. 00000400:00000010:3.0:1788844879.842359:0:68599:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 00000000d192559b. 00000400:00000010:3.0:1788844879.843343:0:68599:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 12 at 00000000d192559b. 00000400:00000010:3.0:1788844879.843355:0:68599:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 12 at 00000000d192559b. 00000400:00000010:3.0:1788844879.844369:0:68599:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 32 at 00000000876b9fa5. 00000001:00000200:3.0:1788844879.844378:0:68599:0:(udsp.c:863:lnet_udsp_get_policy()) Get UDSP at idx = 0 00000001:00000200:3.0:1788844879.844380:0:68599:0:(udsp.c:878:lnet_udsp_get_policy()) Found UDSP (000000007d9b531d) 00000400:00000010:3.0:1788844879.844383:0:68599:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 32 at 00000000876b9fa5. 00000001:02000400:1.0:1788844879.866495:0:68600:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp --if ens2 00000400:00000010:1.0:1788844879.898434:0:68601:0:(api-ni.c:6877:lnet_net_cmd()) alloc '(conf)': 808 at 00000000b9941538. 00000400:00000010:1.0:1788844879.898458:0:68601:0:(api-ni.c:6555:lnet_genl_parse_local_ni()) alloc '(tun)': 172 at 00000000d8a5d020. 00000400:00000010:1.0:1788844879.898471:0:68601:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 0000000016f26155. 00000400:00000010:1.0:1788844879.898476:0:68601:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 560 at 00000000abc33b89. 00000400:00000010:1.0:1788844879.898481:0:68601:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000677873ec. 00000400:00000010:1.0:1788844879.898487:0:68601:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f3ada997. 00000400:00000010:1.0:1788844879.898489:0:68601:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002d37fe28. 00000400:00000010:1.0:1788844879.898491:0:68601:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000cc4a9c2. 00000400:00000010:1.0:1788844879.898495:0:68601:0:(config.c:397:lnet_ni_add_interface()) alloc '(ni->ni_interface)': 5 at 00000000e25e3788. 00000400:00000010:1.0:1788844879.898506:0:68601:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000c1b485da. 00000400:00000010:1.0:1788844879.898509:0:68601:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 000000009c3a85f9. 00000800:00000200:0.0:1788844879.904031:0:68601:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=lo ifindex 1 state 0 ns 000000004dc0f701 00000800:00000200:0.0:1788844879.904042:0:68601:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=lo ifindex 1 state 0 ns 000000004dc0f701 00000800:00000200:0.0:1788844879.904045:0:68601:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=ens2 ifindex 2 state 6 ns 000000004dc0f701 00000800:00000200:0.0:1788844879.904047:0:68601:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=ens2 ifindex 2 state 6 ns 000000004dc0f701 00000800:00000010:0.0:1788844879.904058:0:68601:0:(socklnd.c:2621:ksocknal_startup()) alloc '(net)': 208 at 0000000089ec1266. 00000800:00000200:0.0:1788844879.904074:0:68601:0:(socklnd_modparams.c:295:ksocklnd_lookup_conns_per_peer()) intf ens2 speed -1 00000400:00000200:0.0:1788844879.905173:0:68601:0:(config.c:1657:lnet_get_link_status_locked()) device idx 1 get_link 2 00000400:00000200:0.0:1788844879.905267:0:68601:0:(acceptor.c:209:lnet_acceptor_add_socket()) Added socket on ens2:988 (IP: 192.168.202.29) 00000400:02000000:0.0:1788844879.905286:0:68601:0:(api-ni.c:2823:lnet_startup_lndni()) Added LNI 192.168.202.29@tcp [8/256/0/180] 00000400:00000010:0.0:1788844879.910088:0:68601:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 57 at 00000000266a8074. 00000400:00000010:0.0:1788844879.910127:0:68601:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000efa70c9f. 00000400:00000010:0.0:1788844879.910136:0:68601:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007dec4066. 00000400:00000010:3.0:1788844879.913240:0:68601:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000cdd306f0. 00000400:00000200:3.0:1788844879.913257:0:68601:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000072130235 00000400:00000010:3.0:1788844879.913260:0:68601:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 0000000072130235. 00000400:00000010:3.0:1788844879.913268:0:68601:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 41 at 00000000a1603e8c. 00000400:00000010:3.0:1788844879.913396:0:68601:0:(api-ni.c:6852:lnet_genl_parse_local_ni()) kfreed 'tun': 172 at 00000000d8a5d020. 00000400:00000010:3.0:1788844879.913403:0:68601:0:(api-ni.c:6994:lnet_net_cmd()) kfreed 'conf': 808 at 00000000b9941538. 00000400:00000010:1.0:1788844880.004467:0:68615:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000d384d4c6. 00000400:00000010:1.0:1788844880.004544:0:68615:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000d384d4c6. 00000001:02000400:2.0:1788844880.856558:0:68722:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp1 --if ens2 00000400:00000010:1.0:1788844880.870243:0:68723:0:(api-ni.c:6877:lnet_net_cmd()) alloc '(conf)': 808 at 00000000199c2a60. 00000400:00000010:1.0:1788844880.870274:0:68723:0:(api-ni.c:6555:lnet_genl_parse_local_ni()) alloc '(tun)': 172 at 00000000d1577708. 00000400:00000010:1.0:1788844880.870289:0:68723:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 000000004830a7e6. 00000400:00000010:1.0:1788844880.870300:0:68723:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 560 at 00000000d30daf99. 00000400:00000010:1.0:1788844880.870304:0:68723:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000018b7cfe4. 00000400:00000010:1.0:1788844880.870310:0:68723:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000bf33960d. 00000400:00000010:1.0:1788844880.870312:0:68723:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d63cdfc2. 00000400:00000010:1.0:1788844880.870314:0:68723:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000dd57f84e. 00000400:00000010:1.0:1788844880.870318:0:68723:0:(config.c:397:lnet_ni_add_interface()) alloc '(ni->ni_interface)': 5 at 00000000d384d4c6. 00000800:00000010:1.0:1788844880.870331:0:68723:0:(socklnd.c:2621:ksocknal_startup()) alloc '(net)': 208 at 0000000006ec8db9. 00000800:00000200:1.0:1788844880.870346:0:68723:0:(socklnd_modparams.c:295:ksocklnd_lookup_conns_per_peer()) intf ens2 speed -1 00000400:00000200:1.0:1788844880.870355:0:68723:0:(config.c:1657:lnet_get_link_status_locked()) device idx 1 get_link 2 00000400:00000200:1.0:1788844880.870410:0:68723:0:(acceptor.c:209:lnet_acceptor_add_socket()) Added socket on ens2:988 (IP: 192.168.202.29) 00000400:02000000:1.0:1788844880.870438:0:68723:0:(api-ni.c:2823:lnet_startup_lndni()) Added LNI 192.168.202.29@tcp1 [8/256/0/180] 00000400:00000010:1.0:1788844880.873452:0:68723:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 73 at 00000000417ceea3. 00000400:00000010:1.0:1788844880.873471:0:68723:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000007d2552db. 00000400:00000010:1.0:1788844880.873508:0:68723:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000000e77ec59. 00000400:00000010:1.0:1788844880.873598:0:68723:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000efa70c9f. 00000400:00000200:1.0:1788844880.873606:0:68723:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007dec4066 00000400:00000010:1.0:1788844880.873609:0:68723:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000007dec4066. 00000400:00000010:1.0:1788844880.873615:0:68723:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 57 at 00000000266a8074. 00000400:00000010:1.0:1788844880.873641:0:68723:0:(api-ni.c:6852:lnet_genl_parse_local_ni()) kfreed 'tun': 172 at 00000000d1577708. 00000400:00000010:1.0:1788844880.873649:0:68723:0:(api-ni.c:6994:lnet_net_cmd()) kfreed 'conf': 808 at 00000000199c2a60. 00000400:00000010:3.0:1788844880.914266:0:68728:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000bc565e63. 00000400:00000010:3.0:1788844880.914351:0:68728:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000bc565e63. 00000400:00000010:3.0:1788844881.602679:0:68846:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000bc565e63. 00000400:00000010:3.0:1788844881.602928:0:68846:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000bc565e63. 00000001:02000400:3.0:1788844881.616362:0:68848:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl discover 192.168.202.29@tcp 00000400:00000010:2.0:1788844881.631165:0:68849:0:(peer.c:230:lnet_peer_alloc()) alloc '(lp)': 360 at 000000008c1797eb. 00000400:00000200:2.0:1788844881.631187:0:68849:0:(peer.c:266:lnet_peer_alloc()) 000000008c1797eb nid 192.168.202.29@tcp 00000400:00000010:2.0:1788844881.631192:0:68849:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 0000000024385f93. 00000400:00000200:2.0:1788844881.631195:0:68849:0:(peer.c:203:lnet_peer_net_alloc()) 0000000024385f93 net tcp 00000400:00000010:2.0:1788844881.631198:0:68849:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 000000000ac60c2a. 00000400:00000200:2.0:1788844881.631203:0:68849:0:(peer.c:184:lnet_peer_ni_alloc()) 000000000ac60c2a nid 192.168.202.29@tcp 00000400:00000200:2.0:1788844881.631212:0:68849:0:(peer.c:1804:lnet_peer_attach_peer_ni()) peer 192.168.202.29@tcp NID 192.168.202.29@tcp flags 0x0 00000400:00000200:2.0:1788844881.631216:0:68849:0:(peer.c:2183:lnet_peer_ni_traffic_add()) peer 192.168.202.29@tcp: 0 00000400:00000200:2.0:1788844881.631247:0:68849:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.202.29@tcp: 0 00000400:00000200:2.0:1788844881.631249:0:68849:0:(peer.c:2935:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1788844881.631303:0:68598:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1788844881.631308:0:68598:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.202.29@tcp(000000008c1797eb) state 0x40 00000400:00000010:3.0:1788844881.631314:0:68598:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000084bb106f. 00000400:00000010:3.0:1788844881.631331:0:68598:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000082d5c033. 00000400:00000010:3.0:1788844881.631351:0:68598:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000072429e70. 00000400:00000200:3.0:1788844881.631355:0:68598:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.29@tcp 00000400:00000200:3.0:1788844881.631365:0:68598:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.29@tcp local destination 00000400:00000200:3.0:1788844881.631372:0:68598:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.29@tcp 00000400:00000200:3.0:1788844881.631374:0:68598:0:(lib-move.c:1874:lnet_set_non_mr_pref_nid()) Setting preferred local NID 192.168.202.29@tcp on NMR peer 192.168.202.29@tcp 00000400:00000200:3.0:1788844881.631377:0:68598:0:(peer.c:1171:lnet_peer_ni_set_non_mr_pref_nid()) peer 192.168.202.29@tcp nid 192.168.202.29@tcp: 0 00000400:00000200:3.0:1788844881.631380:0:68598:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [8:8:256:4294967295] 192.168.202.29@tcp LPNI seq info [1:1:8:4294967295] 00000400:00000200:3.0:1788844881.631385:0:68598:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:3.0:1788844881.631390:0:68598:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : GET try# 0 00000800:00000200:3.0:1788844881.631394:0:68598:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.29@tcp 00000800:00000010:3.0:1788844881.631399:0:68598:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000086a58983. 00000800:00000010:3.0:1788844881.631407:0:68598:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 00000000a28c4abc. 00000800:00000010:3.0:1788844881.631410:0:68598:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 00000000aee47713. 00000800:00000200:3.0:1788844881.631414:0:68598:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a28c4abc] -> 12345-192.168.202.29@tcp (2) 00000800:00000200:3.1:1788844881.631417:0:68598:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a28c4abc] -> 12345-192.168.202.29@tcp (2) 00000400:00000200:3.0:1788844881.631430:0:68598:0:(peer.c:4022:lnet_peer_send_ping()) peer 192.168.202.29@tcp 00000400:00000200:3.0:1788844881.631432:0:68598:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.202.29@tcp(000000008c1797eb) state 0x240 rc 0 00000400:00000200:1.0:1788844881.631821:0:68610:0:(acceptor.c:578:lnet_acceptor()) Accepted connection on ens2:988 00000400:00000200:1.0:1788844881.631856:0:68610:0:(acceptor.c:502:lnet_accept()) Accept 192.168.202.29@tcp from 255.3.0.2 00000800:00000010:0.0:1788844881.631858:0:68602:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 00000000b94f5a0a. 00000800:00000010:1.0:1788844881.631861:0:68610:0:(socklnd.c:737:ksocknal_accept()) alloc '(cr)': 32 at 00000000ff34d144. 00000800:00000010:0.0:1788844881.631867:0:68602:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 00000000e37e2096. 00000800:00000010:0.0:1788844881.631876:0:68602:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 000000000f3e66c4. 00000800:00000010:3.0:1788844881.631900:0:68605:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 00000000595b4ee4. 00000800:00000010:3.0:1788844881.631906:0:68605:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 00000000e7c1b137. 00000800:00000010:3.0:1788844881.631929:0:68605:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 00000000f6dc7aea. 00000800:00000200:3.1:1788844881.631935:0:68605:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a28c4abc] -> 12345-192.168.202.29@tcp (2) 00000800:00000200:3.1:1788844881.631940:0:68605:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.202.29@tcp 00000000f6dc7aea deleted 00000800:00000010:3.1:1788844881.631942:0:68605:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 00000000f6dc7aea. 00000800:00000010:2.0:1788844881.631946:0:68602:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 000000000f3e66c4. 00000800:00000200:3.1:1788844881.631947:0:68605:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:3.1:1788844881.631954:0:68605:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000200:3.1:1788844881.631959:0:68605:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000086a58983 type 2, nob 96 niov 1 nkiov 0 00000800:00000200:3.0:1788844881.631964:0:68605:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.202.29@tcp p 3.x 192.168.202.29 -> 192.168.202.29:1023 incarnation:1788844879904060190 sched[0] 00000800:00000010:3.0:1788844881.631970:0:68605:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000302c289e. 00000800:00000010:3.0:1788844881.632030:0:68605:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000302c289e. 00000800:00000010:3.0:1788844881.632040:0:68605:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 00000000e7c1b137. 00000800:00000010:3.0:1788844881.632076:0:68605:0:(socklnd_cb.c:2179:ksocknal_connd()) kfreed 'cr': 32 at 00000000ff34d144. 00000800:00000200:0.0:1788844881.632133:0:68607:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788844881.632137:0:68607:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000086a58983. 00000400:00000200:0.0:1788844881.632142:0:68607:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788844881.632148:0:68607:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: GET: OK 00000400:00000200:0.0:1788844881.632155:0:68607:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1788844881.632158:0:68607:0:(peer.c:3241:lnet_discovery_event_send()) Ping Send to 192.168.202.29@tcp: 0 00000800:00000200:2.1:1788844881.632700:0:68602:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:2.0:1788844881.632708:0:68602:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.202.29@tcp p 3.x 192.168.202.29 -> 192.168.202.29:988 incarnation:1788844879904060190 sched[0] 00000800:00000010:2.0:1788844881.632711:0:68602:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 00000000e37e2096. 00000400:00000200:2.0:1788844881.632809:0:68602:0:(lib-socket.c:438:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.202.29:988 00000800:00000010:2.0:1788844881.632954:0:68602:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 00000000a5ec57bb. 00000800:00000010:2.0:1788844881.632957:0:68602:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 0000000071b0e8a7. 00000800:00000010:2.0:1788844881.632959:0:68602:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000d1b579f2. 00000800:00000010:2.0:1788844881.632978:0:68602:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000d1b579f2. 00000400:00000200:3.0:1788844881.633060:0:68609:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : GET - for me 00000400:00000200:3.0:1788844881.633071:0:68609:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.29@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1788844881.633079:0:68609:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.29@tcp of length 64/272 into md 0xd [1] + 0 00000400:00000200:3.0:1788844881.633087:0:68609:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.29@tcp to NMR: 192.168.202.29@tcp local destination 00000400:00000200:3.0:1788844881.633090:0:68609:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [9:9:256:4294967295] 192.168.202.29@tcp LPNI seq info [2:2:8:4294967295] 00000400:00000200:3.0:1788844881.633095:0:68609:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) 192.168.202.29@tcp : REPLY try# 0 00000800:00000200:3.0:1788844881.633099:0:68609:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.202.29@tcp 00000800:00000010:3.0:1788844881.633103:0:68609:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000086a58983. 00000800:00000200:3.0:1788844881.633106:0:68609:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a28c4abc] -> 12345-192.168.202.29@tcp (4) 00000800:00000200:3.0:1788844881.633110:0:68609:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000200:3.0:1788844881.633112:0:68609:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000086a58983 type 3, nob 160 niov 1 nkiov 1 00000800:00000200:3.0:1788844881.633169:0:68609:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788844881.633172:0:68609:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000086a58983. 00000400:00000200:3.0:1788844881.633175:0:68609:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788844881.633180:0:68609:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: REPLY: OK 00000400:00000200:0.0:1788844881.633194:0:68607:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : REPLY - for me 00000400:00000200:0.0:1788844881.633221:0:68607:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.29@tcp: Reply from 12345-192.168.202.29@tcp of length 64/64 into md 0x11 00000400:00000200:0.0:1788844881.633230:0:68607:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788844881.633235:0:68607:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: REPLY: OK 00000400:00000200:0.0:1788844881.633241:0:68607:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000082d5c033 00000400:00000010:0.0:1788844881.633244:0:68607:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 0000000082d5c033. 00000400:00000200:0.0:1788844881.633248:0:68607:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 3 00000400:00000200:0.0:1788844881.633253:0:68607:0:(peer.c:3119:lnet_discovery_event_reply()) Peer 192.168.202.29@tcp has discovery enabled 00000400:00000200:0.0:1788844881.633256:0:68607:0:(peer.c:3145:lnet_discovery_event_reply()) peer 192.168.202.29@tcp(000000008c1797eb) is MR capable 00000400:00000200:0.0:1788844881.633260:0:68607:0:(peer.c:1196:lnet_peer_ni_clr_non_mr_pref_nid()) peer 192.168.202.29@tcp: 0 00000400:00000200:0.0:1788844881.633264:0:68607:0:(peer.c:3200:lnet_discovery_event_reply()) peer 192.168.202.29@tcp data present 6. state = 0x241 00000400:00000200:0.0:1788844881.633270:0:68607:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.202.29@tcp: -114 00000400:00000200:1.0:1788844881.633279:0:68610:0:(acceptor.c:578:lnet_acceptor()) Accepted connection on ens2:988 00000400:00000200:1.0:1788844881.633296:0:68610:0:(acceptor.c:502:lnet_accept()) Accept 192.168.202.29@tcp from 254.3.0.2 00000400:00000200:3.0:1788844881.633298:0:68598:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000010:1.0:1788844881.633300:0:68610:0:(socklnd.c:737:ksocknal_accept()) alloc '(cr)': 32 at 000000005d4729da. 00000400:00000200:3.0:1788844881.633304:0:68598:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.202.29@tcp(000000008c1797eb) state 0xc1 00000400:00000010:3.0:1788844881.633311:0:68598:0:(peer.c:3474:lnet_peer_merge_data()) alloc '(curnis)': 60 at 00000000302c289e. 00000400:00000010:3.0:1788844881.633314:0:68598:0:(peer.c:3475:lnet_peer_merge_data()) alloc '(addnis)': 72 at 00000000e14976b5. 00000400:00000010:3.0:1788844881.633315:0:68598:0:(peer.c:3476:lnet_peer_merge_data()) alloc '(delnis)': 60 at 00000000bd712de2. 00000400:00000010:3.0:1788844881.633322:0:68598:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 00000000f4e70950. 00000400:00000200:3.0:1788844881.633325:0:68598:0:(peer.c:184:lnet_peer_ni_alloc()) 00000000f4e70950 nid 192.168.202.29@tcp1 00000400:00000010:3.0:1788844881.633327:0:68598:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 00000000fed799a0. 00000400:00000200:3.0:1788844881.633328:0:68598:0:(peer.c:203:lnet_peer_net_alloc()) 00000000fed799a0 net tcp1 00000400:00000200:3.0:1788844881.633332:0:68598:0:(peer.c:1804:lnet_peer_attach_peer_ni()) peer 192.168.202.29@tcp NID 192.168.202.29@tcp1 flags 0x11 00000400:00000010:3.0:1788844881.633336:0:68598:0:(peer.c:3623:lnet_peer_merge_data()) kfreed 'curnis': 60 at 00000000302c289e. 00000400:00000010:3.0:1788844881.633337:0:68598:0:(peer.c:3624:lnet_peer_merge_data()) kfreed 'addnis': 72 at 00000000e14976b5. 00000400:00000010:3.0:1788844881.633339:0:68598:0:(peer.c:3625:lnet_peer_merge_data()) kfreed 'delnis': 60 at 00000000bd712de2. 00000400:00000010:3.0:1788844881.633346:0:68598:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000084bb106f. 00000400:00000200:3.0:1788844881.633349:0:68598:0:(peer.c:3628:lnet_peer_merge_data()) peer 192.168.202.29@tcp (000000008c1797eb): 0 00000400:00000200:3.0:1788844881.633351:0:68598:0:(peer.c:3940:lnet_peer_data_present()) peer 192.168.202.29@tcp(000000008c1797eb): 0. state = 0x141 00000400:00000200:3.0:1788844881.633353:0:68598:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.202.29@tcp(000000008c1797eb) state 0x141 rc 1 00000400:00000200:3.0:1788844881.633355:0:68598:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.202.29@tcp(000000008c1797eb) state 0x141 00000400:00000010:3.0:1788844881.633360:0:68598:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001bba481c. 00000400:00000010:3.0:1788844881.633366:0:68598:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000dfde687d. 00000400:00000200:3.0:1788844881.633369:0:68598:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.29@tcp 00000400:00000200:3.0:1788844881.633375:0:68598:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.29@tcp to MR: 192.168.202.29@tcp local destination 00000400:00000200:3.0:1788844881.633378:0:68598:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [10:10:256:4294967295] 192.168.202.29@tcp LPNI seq info [3:3:8:4294967295] 00000400:00000200:3.0:1788844881.633381:0:68598:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:3.0:1788844881.633384:0:68598:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : PUT try# 0 00000800:00000200:3.0:1788844881.633387:0:68598:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.202.29@tcp 00000800:00000010:3.0:1788844881.633388:0:68598:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000086a58983. 00000800:00000200:3.0:1788844881.633391:0:68598:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a28c4abc] -> 12345-192.168.202.29@tcp (4) 00000800:00000200:3.0:1788844881.633393:0:68598:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000200:3.0:1788844881.633396:0:68598:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000086a58983 type 1, nob 160 niov 1 nkiov 1 00000400:00000200:3.0:1788844881.633405:0:68598:0:(peer.c:4164:lnet_peer_send_push()) peer 192.168.202.29@tcp 00000400:00000200:3.0:1788844881.633407:0:68598:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.202.29@tcp(000000008c1797eb) state 0x541 rc 0 00000800:00000200:0.0:1788844881.633470:0:68609:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788844881.633473:0:68609:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000086a58983. 00000400:00000200:0.0:1788844881.633478:0:68609:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788844881.633485:0:68608:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : PUT - for me 00000400:00000200:0.0:1788844881.633488:0:68609:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: PUT: OK 00000400:00000200:0.0:1788844881.633493:0:68609:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1788844881.633495:0:68608:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.29@tcp of length 64 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1788844881.633499:0:68609:0:(peer.c:3241:lnet_discovery_event_send()) Push Send to 192.168.202.29@tcp: 0 00000400:00000200:3.0:1788844881.633502:0:68608:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 0 from 12345-192.168.202.29@tcp of length 64/64 into md 0x5 [1] + 0 00000400:00000010:3.0:1788844881.633506:0:68608:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000074a32fc1. 00000400:00000200:3.0:1788844881.633509:0:68608:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000008812e9f4 00000400:00000200:3.0:1788844881.633513:0:68608:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788844881.633516:0:68608:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: PUT: OK 00000400:00000200:3.0:1788844881.633521:0:68608:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000008812e9f4 00000400:00000010:3.0:1788844881.633522:0:68608:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000008812e9f4. 00000400:00000200:3.0:1788844881.633525:0:68608:0:(api-ni.c:2393:lnet_push_target_event_handler()) type 2 status 0 unlinked 1 00000400:00000200:3.0:1788844881.633530:0:68608:0:(peer.c:2778:lnet_peer_push_event()) peer 192.168.202.29@tcp(000000008c1797eb) is MR 00000400:00000010:3.0:1788844881.633538:0:68608:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 73 at 00000000e14976b5. 00000400:00000200:3.0:1788844881.633541:0:68608:0:(peer.c:2852:lnet_peer_push_event()) Received Push 192.168.202.29@tcp 6 00000400:00000200:3.0:1788844881.633543:0:68608:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.202.29@tcp: -114 00000400:00000200:3.0:1788844881.633555:0:68608:0:(lib-md.c:96:lnet_cpt_of_md()) Couldn't resolve first page of md 0000000000000000 with offset 0 00000400:00000200:3.0:1788844881.633558:0:68608:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.29@tcp to MR: 192.168.202.29@tcp local destination 00000400:00000200:3.0:1788844881.633560:0:68608:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [11:11:256:4294967295] 192.168.202.29@tcp LPNI seq info [4:4:8:4294967295] 00000400:00000200:3.0:1788844881.633565:0:68608:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) 192.168.202.29@tcp : ACK try# 0 00000800:00000200:3.0:1788844881.633567:0:68608:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 1 frags to 12345-192.168.202.29@tcp 00000800:00000010:3.0:1788844881.633569:0:68608:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005620bd8e. 00000800:00000200:3.0:1788844881.633571:0:68608:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a28c4abc] -> 12345-192.168.202.29@tcp (4) 00000800:00000200:3.0:1788844881.633573:0:68608:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000200:3.0:1788844881.633575:0:68608:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005620bd8e type 0, nob 96 niov 1 nkiov 0 00000800:00000200:3.0:1788844881.633604:0:68608:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788844881.633606:0:68608:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005620bd8e. 00000400:00000200:3.0:1788844881.633608:0:68608:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788844881.633610:0:68608:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: ACK: OK 00000400:00000200:0.0:1788844881.633617:0:68598:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1788844881.633622:0:68608:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : ACK - for me 00000400:00000010:0.0:1788844881.633624:0:68598:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a495e614. 00000400:00000200:3.0:1788844881.633630:0:68608:0:(lib-move.c:4539:lnet_parse_ack()) 192.168.202.29@tcp: ACK from 12345-192.168.202.29@tcp into md 0x15 00000400:00000200:3.0:1788844881.633632:0:68608:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788844881.633634:0:68608:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: ACK: OK 00000400:00000010:0.0:1788844881.633645:0:68598:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000097e06605. 00000400:00000200:3.0:1788844881.633649:0:68608:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001bba481c 00000400:00000010:3.0:1788844881.633651:0:68608:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000001bba481c. 00000400:00000200:3.0:1788844881.633653:0:68608:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 4 00000400:00000200:3.0:1788844881.633655:0:68608:0:(peer.c:3007:lnet_discovery_event_ack()) peer 192.168.202.29@tcp ev->status 0 00000400:00000200:0.0:1788844881.633656:0:68598:0:(api-ni.c:2383:lnet_push_target_post()) posted push target 0000000024ee44a7 00000400:00000200:3.0:1788844881.633657:0:68608:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.202.29@tcp: -114 00000400:00000200:0.0:1788844881.633669:0:68598:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.202.29@tcp(000000008c1797eb) state 0xc1 00000400:00000010:0.0:1788844881.633674:0:68598:0:(peer.c:3474:lnet_peer_merge_data()) alloc '(curnis)': 60 at 0000000071598e2c. 00000400:00000010:0.0:1788844881.633677:0:68598:0:(peer.c:3475:lnet_peer_merge_data()) alloc '(addnis)': 72 at 00000000d19f0657. 00000400:00000010:0.0:1788844881.633678:0:68598:0:(peer.c:3476:lnet_peer_merge_data()) alloc '(delnis)': 60 at 00000000af6c91a9. 00000400:00000010:0.0:1788844881.633682:0:68598:0:(peer.c:3623:lnet_peer_merge_data()) kfreed 'curnis': 60 at 0000000071598e2c. 00000400:00000010:0.0:1788844881.633685:0:68598:0:(peer.c:3624:lnet_peer_merge_data()) kfreed 'addnis': 72 at 00000000d19f0657. 00000400:00000010:0.0:1788844881.633687:0:68598:0:(peer.c:3625:lnet_peer_merge_data()) kfreed 'delnis': 60 at 00000000af6c91a9. 00000400:00000010:0.0:1788844881.633689:0:68598:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 73 at 00000000e14976b5. 00000400:00000200:0.0:1788844881.633692:0:68598:0:(peer.c:3628:lnet_peer_merge_data()) peer 192.168.202.29@tcp (000000008c1797eb): 0 00000400:00000200:0.0:1788844881.633694:0:68598:0:(peer.c:3940:lnet_peer_data_present()) peer 192.168.202.29@tcp(000000008c1797eb): 0. state = 0x141 00000400:00000200:0.0:1788844881.633697:0:68598:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.202.29@tcp(000000008c1797eb) state 0x141 rc 1 00000400:00000200:0.0:1788844881.633700:0:68598:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.202.29@tcp(000000008c1797eb) state 0x141 00000400:00000200:0.0:1788844881.633702:0:68598:0:(peer.c:4082:lnet_peer_discovered()) peer 192.168.202.29@tcp 00000400:00000200:0.0:1788844881.633704:0:68598:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.202.29@tcp(000000008c1797eb) state 0x111 rc 0 00000400:00000200:0.0:1788844881.633708:0:68598:0:(peer.c:2629:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 192.168.202.29@tcp 00000400:00000200:3.0:1788844881.633833:0:68849:0:(peer.c:2984:lnet_discover_peer_locked()) peer 192.168.202.29@tcp NID 192.168.202.29@tcp: 0. discovery complete 00000800:00000010:2.0:1788844881.635710:0:68603:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 000000006ade1324. 00000800:00000010:2.0:1788844881.635732:0:68603:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 00000000f4eacea9. 00000800:00000010:2.0:1788844881.635809:0:68603:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 00000000c26ceead. 00000800:00000200:2.1:1788844881.635820:0:68603:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a28c4abc] -> 12345-192.168.202.29@tcp (4) 00000800:00000200:2.1:1788844881.635826:0:68603:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.202.29@tcp 00000000c26ceead deleted 00000800:00000010:2.1:1788844881.635829:0:68603:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 00000000c26ceead. 00000800:00000200:2.1:1788844881.635835:0:68603:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 3, ksnr_connected a ksnr_max_conns 1 00000800:00000200:2.0:1788844881.635844:0:68603:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.202.29@tcp p 3.x 192.168.202.29 -> 192.168.202.29:1022 incarnation:1788844879904060190 sched[0] 00000800:00000010:3.0:1788844881.635892:0:68603:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000bd712de2. 00000800:00000010:3.0:1788844881.636011:0:68603:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000bd712de2. 00000800:00000010:3.0:1788844881.636015:0:68603:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 00000000f4eacea9. 00000800:00000200:2.1:1788844881.636025:0:68602:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 2, ksnr_connected e ksnr_max_conns 1 00000800:00000200:2.0:1788844881.636033:0:68602:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.202.29@tcp p 3.x 192.168.202.29 -> 192.168.202.29:988 incarnation:1788844879904060190 sched[0] 00000800:00000010:2.0:1788844881.636037:0:68602:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 0000000071b0e8a7. 00000800:00000010:3.0:1788844881.636052:0:68603:0:(socklnd_cb.c:2179:ksocknal_connd()) kfreed 'cr': 32 at 000000005d4729da. 00000001:02000400:3.0:1788844881.641423:0:68850:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 0 00000400:00000010:3.0:1788844881.667009:0:68853:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 12 at 00000000d192559b. 00000400:00000010:3.0:1788844881.667030:0:68853:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 12 at 00000000d192559b. 00000001:02000400:3.0:1788844881.698338:0:68855:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set max_recovery_ping_interval 4 00000001:02000400:0.0:1788844881.742730:0:68858:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net_drop add -s *@tcp -d *@tcp -m GET -r 1 -e local_error 00000400:00000200:1.0:1788844881.762344:0:68859:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844881.762358:0:68859:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844881.762372:0:68859:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp 00000400:00000200:1.0:1788844881.762374:0:68859:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844881.762377:0:68859:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp 00000400:00000200:1.0:1788844881.762379:0:68859:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844881.762382:0:68859:0:(api-ni.c:3207:nla_strnid()) -> 00000400:00000200:1.0:1788844881.762383:0:68859:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844881.762385:0:68859:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844881.762387:0:68859:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844881.762388:0:68859:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844881.762396:0:68859:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844881.762397:0:68859:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844881.762399:0:68859:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000010:1.0:1788844881.762404:0:68859:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 00000000ac329c1f. 00000400:00000200:1.0:1788844881.762414:0:68859:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp, dst 255.255.255.255@tcp, rate 1, interval 0 00000001:02000400:3.0:1788844881.777448:0:68860:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net_drop add -s *@tcp1 -d *@tcp1 -m GET -r 1 -e local_error 00000400:00000200:2.0:1788844881.801689:0:68861:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844881.801702:0:68861:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844881.801731:0:68861:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1 00000400:00000200:2.0:1788844881.801733:0:68861:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844881.801737:0:68861:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1 00000400:00000200:2.0:1788844881.801738:0:68861:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844881.801747:0:68861:0:(api-ni.c:3207:nla_strnid()) -> 00000400:00000200:2.0:1788844881.801748:0:68861:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844881.801750:0:68861:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844881.801752:0:68861:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844881.801753:0:68861:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844881.801755:0:68861:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844881.801757:0:68861:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844881.801760:0:68861:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000010:2.0:1788844881.801765:0:68861:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 0000000038c91132. 00000400:00000200:2.0:1788844881.801776:0:68861:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp1, dst 255.255.255.255@tcp1, rate 1, interval 0 00000001:02000400:0.0:1788844881.812411:0:68862:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net_drop add -s *@tcp -d *@tcp -r 1 00000400:00000200:3.0:1788844881.830695:0:68863:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844881.830709:0:68863:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844881.830736:0:68863:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp 00000400:00000200:3.0:1788844881.830738:0:68863:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844881.830740:0:68863:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp 00000400:00000200:3.0:1788844881.830742:0:68863:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844881.830745:0:68863:0:(api-ni.c:3207:nla_strnid()) -> 00000400:00000200:3.0:1788844881.830746:0:68863:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844881.830747:0:68863:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844881.830749:0:68863:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844881.830750:0:68863:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844881.830752:0:68863:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844881.830753:0:68863:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844881.830755:0:68863:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000010:3.0:1788844881.830761:0:68863:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 0000000045a17cc2. 00000400:00000200:3.0:1788844881.830770:0:68863:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp, dst 255.255.255.255@tcp, rate 1, interval 0 00000001:02000400:0.0:1788844881.841523:0:68864:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net_drop add -s *@tcp1 -d *@tcp1 -r 1 00000400:00000200:1.0:1788844881.857389:0:68865:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844881.857403:0:68865:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844881.857422:0:68865:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1 00000400:00000200:1.0:1788844881.857425:0:68865:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844881.857428:0:68865:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1 00000400:00000200:1.0:1788844881.857429:0:68865:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844881.857432:0:68865:0:(api-ni.c:3207:nla_strnid()) -> 00000400:00000200:1.0:1788844881.857433:0:68865:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844881.857434:0:68865:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844881.857436:0:68865:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844881.857437:0:68865:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844881.857439:0:68865:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844881.857441:0:68865:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844881.857443:0:68865:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000010:1.0:1788844881.857448:0:68865:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 000000002221eecc. 00000400:00000200:1.0:1788844881.857460:0:68865:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp1, dst 255.255.255.255@tcp1, rate 1, interval 0 00000001:02000400:0.0:1788844881.872357:0:68866:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net set --health 0 --nid 192.168.202.29@tcp 00000400:00000010:3.0:1788844881.892431:0:68867:0:(api-ni.c:6877:lnet_net_cmd()) alloc '(conf)': 808 at 000000000a4cc18f. 00000400:00000010:3.0:1788844881.892456:0:68867:0:(api-ni.c:6555:lnet_genl_parse_local_ni()) alloc '(tun)': 172 at 00000000e7c1b137. 00000400:00000200:3.0:1788844881.892461:0:68867:0:(api-ni.c:6621:lnet_genl_parse_local_ni()) Requested NID 192.168.202.29@tcp 00000400:00020000:3.0:1788844881.892479:0:68867:0:(api-ni.c:4473:lnet_ni_set_healthv()) manually adding local NI 192.168.202.29@tcp to recovery 00000400:00000200:3.0:1788844881.897649:0:68867:0:(lib-lnet.h:200:lnet_ni_set_status_locked()) ni 192.168.202.29@tcp status changed from 0x15aac0de to 0xdeadface 00000400:00000010:3.0:1788844881.897661:0:68867:0:(api-ni.c:6852:lnet_genl_parse_local_ni()) kfreed 'tun': 172 at 00000000e7c1b137. 00000400:00000010:3.0:1788844881.897673:0:68867:0:(api-ni.c:6994:lnet_net_cmd()) kfreed 'conf': 808 at 000000000a4cc18f. 00000400:00000010:0.0:1788844881.956501:0:68873:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000996d6136. 00000400:00000010:0.0:1788844881.956613:0:68873:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000996d6136. 00000400:00000200:2.0:1788844882.904967:0:68597:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.29@tcp 00000400:00000010:2.0:1788844882.904982:0:68597:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000c496f7da. 00000400:00000010:2.0:1788844882.904994:0:68597:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000b03adc18. 00000400:00000010:2.0:1788844882.905035:0:68597:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a767a2ab. 00000400:00000010:2.0:1788844882.905062:0:68597:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000fcdf6c1f. 00000400:00000200:2.0:1788844882.905067:0:68597:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.29@tcp 00000400:00000200:2.0:1788844882.905084:0:68597:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.29@tcp local destination 00000400:00000200:2.0:1788844882.905091:0:68597:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.29@tcp 00000400:00000200:2.0:1788844882.905094:0:68597:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [12:12:256:4294967295] 192.168.202.29@tcp LPNI seq info [5:5:8:4294967295] 00000400:00000200:2.0:1788844882.905102:0:68597:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:2.0:1788844882.905109:0:68597:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : GET try# 0 00000800:00000200:2.0:1788844882.905115:0:68597:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.29@tcp 00000800:00000010:2.0:1788844882.905121:0:68597:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000085e5a865. 00000800:00000200:2.0:1788844882.905128:0:68597:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a28c4abc] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:2.0:1788844882.905137:0:68597:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000200:2.0:1788844882.905142:0:68597:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000085e5a865 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:2.0:1788844882.905170:0:68597:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.29@tcp added to recovery queue. ping count: 1 next ping: 1448 health :0 00000400:00000200:3.0:1788844882.905651:0:68609:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 1 00000400:00000200:3.0:1788844882.905659:0:68609:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.202.29@tcp(192.168.202.29@tcp)->dst 192.168.202.29@tcp: GET simulate health error: LOCAL_ERROR 00000800:02000400:3.0:1788844882.905662:0:68609:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.202.29: rc = -22 00000800:00000200:3.0:1788844882.908573:0:68609:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [00000000b94f5a0a] Error -22 on write to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000010:3.0:1788844882.908587:0:68609:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000085e5a865. 00000400:00000200:3.0:1788844882.908620:0:68609:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 5 00000400:00000200:3.0:1788844882.908627:0:68609:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: GET: LOCAL_ERROR 00000400:00000200:3.0:1788844882.908638:0:68609:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5 00000400:00000200:3.0:1788844882.908642:0:68609:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.29@tcp recovery message sent unsuccessfully:-5 00000400:00000200:3.0:1788844882.908646:0:68609:0:(lib-move.c:4071:lnet_handle_recovery_reply()) local NI (192.168.202.29@tcp) recovery failed with -5 00000400:00000010:3.0:1788844883.065543:0:68883:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000bc565e63. 00000400:00000010:3.0:1788844883.065679:0:68883:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000bc565e63. 00000400:00000200:2.0:1788844883.927957:0:68597:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a767a2ab 00000400:00000010:2.0:1788844883.927974:0:68597:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000a767a2ab. 00000400:00000200:2.0:1788844883.927985:0:68597:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0 00000400:00000200:2.0:1788844883.927993:0:68597:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.202.29@tcp recovery ping unlinked 00000400:00000010:2.0:1788844883.927998:0:68597:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000c496f7da. 00000400:00000010:2.0:1788844883.928005:0:68597:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000b03adc18. 00000400:00000010:2.0:1788844884.212708:0:68893:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000a244a972. 00000400:00000010:2.0:1788844884.213110:0:68893:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000a244a972. 00000400:00000200:2.0:1788844884.951968:0:68597:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.29@tcp 00000400:00000010:2.0:1788844884.951985:0:68597:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000041114b90. 00000400:00000010:2.0:1788844884.951998:0:68597:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000081735f1b. 00000400:00000010:2.0:1788844884.952006:0:68597:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a767a2ab. 00000400:00000010:2.0:1788844884.952017:0:68597:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000008cfc270a. 00000400:00000200:2.0:1788844884.952022:0:68597:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.29@tcp 00000400:00000200:2.0:1788844884.952040:0:68597:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.29@tcp local destination 00000400:00000200:2.0:1788844884.952048:0:68597:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.29@tcp 00000400:00000200:2.0:1788844884.952052:0:68597:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [13:13:256:4294967295] 192.168.202.29@tcp LPNI seq info [6:6:8:4294967295] 00000400:00000200:2.0:1788844884.952060:0:68597:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:2.0:1788844884.952067:0:68597:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : GET try# 0 00000800:00000200:2.0:1788844884.952074:0:68597:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.29@tcp 00000800:00000010:2.0:1788844884.952080:0:68597:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000b8270d8a. 00000800:00000200:2.0:1788844884.952088:0:68597:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a28c4abc] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:2.0:1788844884.952097:0:68597:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000200:2.0:1788844884.952102:0:68597:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000b8270d8a type 2, nob 96 niov 1 nkiov 0 00000400:00000200:2.0:1788844884.952137:0:68597:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.29@tcp added to recovery queue. ping count: 2 next ping: 1452 health :0 00000400:00000200:3.0:1788844884.952278:0:68607:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 2 00000400:00000200:3.0:1788844884.952286:0:68607:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.202.29@tcp(192.168.202.29@tcp)->dst 192.168.202.29@tcp: GET simulate health error: LOCAL_ERROR 00000800:02000400:3.0:1788844884.952289:0:68607:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.202.29: rc = -22 00000800:00000200:3.0:1788844884.952293:0:68607:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [00000000595b4ee4] Error -22 on write to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000010:3.0:1788844884.952298:0:68607:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000b8270d8a. 00000400:00000200:3.0:1788844884.952315:0:68607:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 5 00000400:00000200:3.0:1788844884.952320:0:68607:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: GET: LOCAL_ERROR 00000400:00000200:3.0:1788844884.952328:0:68607:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5 00000400:00000200:3.0:1788844884.952333:0:68607:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.29@tcp recovery message sent unsuccessfully:-5 00000400:00000200:3.0:1788844884.952338:0:68607:0:(lib-move.c:4071:lnet_handle_recovery_reply()) local NI (192.168.202.29@tcp) recovery failed with -5 00000400:00000010:1.0:1788844885.334214:0:68903:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 0000000094fe76cd. 00000400:00000010:1.0:1788844885.334325:0:68903:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 0000000094fe76cd. 00000400:00000010:2.0:1788844885.378422:0:68907:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000a244a972. 00000400:00000010:2.0:1788844885.378533:0:68907:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000a244a972. 00000400:00000200:2.0:1788844885.438568:0:68914:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 0 00000400:00000010:2.0:1788844885.438578:0:68914:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 0000000016262800. 00000400:00000200:2.0:1788844885.438589:0:68914:0:(api-ni.c:9567:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:2.0:1788844885.438608:0:68914:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 000000008ac2a6bf. 00000400:00000200:2.0:1788844885.438616:0:68914:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:2.0:1788844885.438620:0:68914:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 000000008ac2a6bf. 00000400:00000010:2.0:1788844885.438630:0:68914:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 0000000016262800. 00000001:02000400:1.0:1788844885.464733:0:68916:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -l 00000400:00000200:0.0:1788844885.480478:0:68917:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 0 00000400:00000010:0.0:1788844885.480487:0:68917:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000a1f304c6. 00000400:00000200:0.0:1788844885.480496:0:68917:0:(api-ni.c:9567:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:0.0:1788844885.480512:0:68917:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000386984a5. 00000400:00000200:0.0:1788844885.480519:0:68917:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:0.0:1788844885.480523:0:68917:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000386984a5. 00000400:00000010:0.0:1788844885.480539:0:68917:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000a1f304c6. 00000400:00000010:2.0:1788844885.523983:0:68923:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000a244a972. 00000400:00000010:2.0:1788844885.524096:0:68923:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000a244a972. 00000400:00000200:2.0:1788844885.976132:0:68597:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a767a2ab 00000400:00000010:2.0:1788844885.976160:0:68597:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000a767a2ab. 00000400:00000200:2.0:1788844885.976168:0:68597:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0 00000400:00000200:2.0:1788844885.976175:0:68597:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.202.29@tcp recovery ping unlinked 00000400:00000010:2.0:1788844885.976179:0:68597:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000041114b90. 00000400:00000010:2.0:1788844885.976184:0:68597:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000081735f1b. 00000400:00000010:2.0:1788844886.588825:0:68934:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000a244a972. 00000400:00000010:2.0:1788844886.588948:0:68934:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000a244a972. 00000400:00000010:1.0:1788844887.679971:0:68944:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 0000000094fe76cd. 00000400:00000010:1.0:1788844887.680081:0:68944:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 0000000094fe76cd. 00000400:00000010:3.0:1788844888.817386:0:68954:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000bc565e63. 00000400:00000010:3.0:1788844888.817510:0:68954:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000bc565e63. 00000400:00000200:2.0:1788844889.047919:0:68597:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.29@tcp 00000400:00000010:2.0:1788844889.047931:0:68597:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000001b31a72. 00000400:00000010:2.0:1788844889.047941:0:68597:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000fa760abd. 00000400:00000010:2.0:1788844889.047946:0:68597:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a767a2ab. 00000400:00000010:2.0:1788844889.047955:0:68597:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000001ee9e81f. 00000400:00000200:2.0:1788844889.047959:0:68597:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.29@tcp 00000400:00000200:2.0:1788844889.047975:0:68597:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.29@tcp local destination 00000400:00000200:2.0:1788844889.047981:0:68597:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.29@tcp 00000400:00000200:2.0:1788844889.047986:0:68597:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [14:14:256:4294967295] 192.168.202.29@tcp LPNI seq info [7:7:8:4294967295] 00000400:00000200:2.0:1788844889.047992:0:68597:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:2.0:1788844889.048000:0:68597:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : GET try# 0 00000800:00000200:2.0:1788844889.048005:0:68597:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.29@tcp 00000800:00000010:2.0:1788844889.048010:0:68597:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000007ace9e47. 00000800:00000200:2.0:1788844889.048016:0:68597:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a28c4abc] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:2.0:1788844889.048024:0:68597:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000200:2.0:1788844889.048028:0:68597:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000007ace9e47 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:2.0:1788844889.048054:0:68597:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.29@tcp added to recovery queue. ping count: 3 next ping: 1456 health :0 00000400:00000200:1.0:1788844889.048151:0:68608:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 3 00000400:00000200:1.0:1788844889.048157:0:68608:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.202.29@tcp(192.168.202.29@tcp)->dst 192.168.202.29@tcp: GET simulate health error: LOCAL_ERROR 00000800:02000400:1.0:1788844889.048161:0:68608:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.202.29: rc = -22 00000800:00000200:1.0:1788844889.055191:0:68608:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [00000000b94f5a0a] Error -22 on write to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000010:1.0:1788844889.055202:0:68608:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000007ace9e47. 00000400:00000200:1.0:1788844889.055210:0:68608:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 5 00000400:00000200:1.0:1788844889.055215:0:68608:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: GET: LOCAL_ERROR 00000400:00000200:1.0:1788844889.055225:0:68608:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5 00000400:00000200:1.0:1788844889.055228:0:68608:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.29@tcp recovery message sent unsuccessfully:-5 00000400:00000200:1.0:1788844889.055250:0:68608:0:(lib-move.c:4071:lnet_handle_recovery_reply()) local NI (192.168.202.29@tcp) recovery failed with -5 00000400:00000010:3.0:1788844889.935243:0:68964:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000bc565e63. 00000400:00000010:3.0:1788844889.937173:0:68964:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000bc565e63. 00000400:00000200:2.0:1788844890.076545:0:68597:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a767a2ab 00000400:00000010:2.0:1788844890.076556:0:68597:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000a767a2ab. 00000400:00000200:2.0:1788844890.076564:0:68597:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0 00000400:00000200:2.0:1788844890.076571:0:68597:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.202.29@tcp recovery ping unlinked 00000400:00000010:2.0:1788844890.076584:0:68597:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000001b31a72. 00000400:00000010:2.0:1788844890.076590:0:68597:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000fa760abd. 00000400:00000010:3.0:1788844891.061076:0:68975:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000bc565e63. 00000400:00000010:3.0:1788844891.061190:0:68975:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000bc565e63. 00000400:00000010:1.0:1788844892.199112:0:68985:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 0000000094fe76cd. 00000400:00000010:1.0:1788844892.199219:0:68985:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 0000000094fe76cd. 00000400:00000200:0.0:1788844893.144403:0:68597:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.29@tcp 00000400:00000010:0.0:1788844893.144414:0:68597:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000053ef6c0d. 00000400:00000010:0.0:1788844893.144438:0:68597:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000043a5ae6. 00000400:00000010:0.0:1788844893.144445:0:68597:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003ed6e096. 00000400:00000010:0.0:1788844893.144453:0:68597:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000fa541c26. 00000400:00000200:0.0:1788844893.144458:0:68597:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.29@tcp 00000400:00000200:0.0:1788844893.144473:0:68597:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.29@tcp local destination 00000400:00000200:0.0:1788844893.144480:0:68597:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.29@tcp 00000400:00000200:0.0:1788844893.144484:0:68597:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [15:15:256:4294967295] 192.168.202.29@tcp LPNI seq info [8:8:8:4294967295] 00000400:00000200:0.0:1788844893.144490:0:68597:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:0.0:1788844893.144498:0:68597:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : GET try# 0 00000800:00000200:0.0:1788844893.144504:0:68597:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.29@tcp 00000800:00000010:0.0:1788844893.144511:0:68597:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000306fec4d. 00000800:00000200:0.0:1788844893.144517:0:68597:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a28c4abc] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:0.0:1788844893.144524:0:68597:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000200:0.0:1788844893.144529:0:68597:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000306fec4d type 2, nob 96 niov 1 nkiov 0 00000400:00000200:0.0:1788844893.144554:0:68597:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.29@tcp added to recovery queue. ping count: 4 next ping: 1460 health :0 00000400:00000200:3.0:1788844893.144601:0:68609:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 4 00000400:00000200:3.0:1788844893.144623:0:68609:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.202.29@tcp(192.168.202.29@tcp)->dst 192.168.202.29@tcp: GET simulate health error: LOCAL_ERROR 00000800:02000400:3.0:1788844893.144627:0:68609:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.202.29: rc = -22 00000800:00000200:3.0:1788844893.144630:0:68609:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [00000000595b4ee4] Error -22 on write to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000010:3.0:1788844893.144634:0:68609:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000306fec4d. 00000400:00000200:3.0:1788844893.144640:0:68609:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 5 00000400:00000200:3.0:1788844893.144650:0:68609:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: GET: LOCAL_ERROR 00000400:00000200:3.0:1788844893.144657:0:68609:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5 00000400:00000200:3.0:1788844893.144659:0:68609:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.29@tcp recovery message sent unsuccessfully:-5 00000400:00000200:3.0:1788844893.144663:0:68609:0:(lib-move.c:4071:lnet_handle_recovery_reply()) local NI (192.168.202.29@tcp) recovery failed with -5 00000400:00000010:3.0:1788844893.370956:0:68995:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000bc565e63. 00000400:00000010:3.0:1788844893.371067:0:68995:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000bc565e63. 00000400:00000010:3.0:1788844893.435451:0:68999:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000bc565e63. 00000400:00000010:3.0:1788844893.435561:0:68999:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000bc565e63. 00000400:00000200:3.0:1788844893.513480:0:69006:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 0 00000400:00000010:3.0:1788844893.513489:0:69006:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000659cb8f1. 00000400:00000200:3.0:1788844893.513499:0:69006:0:(api-ni.c:9567:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:3.0:1788844893.513518:0:69006:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 000000007978d37a. 00000400:00000200:3.0:1788844893.513525:0:69006:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:3.0:1788844893.513529:0:69006:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 000000007978d37a. 00000400:00000010:3.0:1788844893.513537:0:69006:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000659cb8f1. 00000001:02000400:1.0:1788844893.555216:0:69008:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -l 00000400:00000200:2.0:1788844893.625387:0:69009:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 0 00000400:00000010:2.0:1788844893.625401:0:69009:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000aa794c31. 00000400:00000200:2.0:1788844893.625412:0:69009:0:(api-ni.c:9567:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:2.0:1788844893.625429:0:69009:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000d5dded7f. 00000400:00000200:2.0:1788844893.625436:0:69009:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:2.0:1788844893.625441:0:69009:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000d5dded7f. 00000400:00000010:2.0:1788844893.625449:0:69009:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000aa794c31. 00000001:02000400:0.0:1788844893.666119:0:69010:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net_drop del -a 00000400:00000200:3.0:1788844893.706166:0:69011:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844893.706178:0:69011:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844893.706183:0:69011:0:(api-ni.c:3207:nla_strnid()) -> 00000400:00000200:3.0:1788844893.706184:0:69011:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844893.706186:0:69011:0:(api-ni.c:3207:nla_strnid()) -> 00000400:00000200:3.0:1788844893.706189:0:69011:0:(net_fault.c:254:lnet_drop_rule_del()) src dst 00000400:00000200:3.0:1788844893.706198:0:69011:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp->dst: 255.255.255.255@tcp (1/1, 0) 00000400:00000010:3.0:1788844893.706200:0:69011:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 00000000ac329c1f. 00000400:00000200:3.0:1788844893.706209:0:69011:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp1->dst: 255.255.255.255@tcp1 (1/1, 0) 00000400:00000010:3.0:1788844893.706211:0:69011:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 0000000038c91132. 00000400:00000200:3.0:1788844893.706214:0:69011:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp->dst: 255.255.255.255@tcp (1/1, 0) 00000400:00000010:3.0:1788844893.706216:0:69011:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 0000000045a17cc2. 00000400:00000200:3.0:1788844893.706219:0:69011:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp1->dst: 255.255.255.255@tcp1 (1/1, 0) 00000400:00000010:3.0:1788844893.706221:0:69011:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 000000002221eecc. 00000001:02000400:1.0:1788844893.730227:0:69012:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set max_recovery_ping_interval 900 00000400:00000200:2.0:1788844894.174023:0:68597:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003ed6e096 00000400:00000010:2.0:1788844894.174033:0:68597:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000003ed6e096. 00000400:00000200:2.0:1788844894.174039:0:68597:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0 00000400:00000200:2.0:1788844894.174047:0:68597:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.202.29@tcp recovery ping unlinked 00000400:00000010:2.0:1788844894.174051:0:68597:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000053ef6c0d. 00000400:00000010:2.0:1788844894.174056:0:68597:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000043a5ae6. 00000400:00000200:0.0:1788844896.217137:0:68597:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.29@tcp 00000400:00000010:0.0:1788844896.217146:0:68597:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000045b70078. 00000400:00000010:0.0:1788844896.217155:0:68597:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000e6d51ddb. 00000400:00000010:0.0:1788844896.217161:0:68597:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001a2ddf4c. 00000400:00000010:0.0:1788844896.217168:0:68597:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000000070272c. 00000400:00000200:0.0:1788844896.217171:0:68597:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.29@tcp 00000400:00000200:0.0:1788844896.217185:0:68597:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.29@tcp local destination 00000400:00000200:0.0:1788844896.217191:0:68597:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.29@tcp 00000400:00000200:0.0:1788844896.217196:0:68597:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [16:16:256:4294967295] 192.168.202.29@tcp LPNI seq info [9:9:8:4294967295] 00000400:00000200:0.0:1788844896.217202:0:68597:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:0.0:1788844896.217209:0:68597:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : GET try# 0 00000800:00000200:0.0:1788844896.217214:0:68597:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.29@tcp 00000800:00000010:0.0:1788844896.217219:0:68597:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000d57ee5d9. 00000800:00000200:0.0:1788844896.217237:0:68597:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a28c4abc] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:0.0:1788844896.217245:0:68597:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000200:0.0:1788844896.217249:0:68597:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000d57ee5d9 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:0.0:1788844896.217273:0:68597:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.29@tcp added to recovery queue. ping count: 5 next ping: 1492 health :0 00000800:00000200:3.0:1788844896.217674:0:68607:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788844896.217679:0:68607:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000d57ee5d9. 00000400:00000200:3.0:1788844896.217683:0:68607:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788844896.217688:0:68607:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: GET: OK 00000400:00000200:3.0:1788844896.217693:0:68607:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:3.0:1788844896.217703:0:68607:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.29@tcp recovery message sent successfully:0 00000400:00000200:3.0:1788844896.217732:0:68607:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : GET - for me 00000400:00000200:3.0:1788844896.217742:0:68607:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.29@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1788844896.217749:0:68607:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.29@tcp of length 64/272 into md 0xd [1] + 0 00000400:00000200:3.0:1788844896.217757:0:68607:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.29@tcp to MR: 192.168.202.29@tcp local destination 00000400:00000200:3.0:1788844896.217762:0:68607:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [17:17:256:4294967295] 192.168.202.29@tcp LPNI seq info [10:10:8:4294967295] 00000400:00000200:3.0:1788844896.217770:0:68607:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) 192.168.202.29@tcp : REPLY try# 0 00000800:00000200:3.0:1788844896.217836:0:68607:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.202.29@tcp 00000800:00000010:3.0:1788844896.217841:0:68607:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c2cabb8c. 00000800:00000200:3.0:1788844896.217845:0:68607:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a28c4abc] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:3.0:1788844896.217850:0:68607:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000200:3.0:1788844896.217853:0:68607:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c2cabb8c type 3, nob 160 niov 1 nkiov 1 00000800:00000200:3.0:1788844896.217905:0:68607:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788844896.217907:0:68607:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c2cabb8c. 00000400:00000200:3.0:1788844896.217910:0:68607:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788844896.217914:0:68607:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: REPLY: OK 00000400:00000200:3.0:1788844896.217929:0:68607:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : REPLY - for me 00000400:00000200:3.0:1788844896.217937:0:68607:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.29@tcp: Reply from 12345-192.168.202.29@tcp of length 64/64 into md 0x2d 00000400:00000200:3.0:1788844896.217943:0:68607:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788844896.217946:0:68607:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: REPLY: OK 00000400:00000200:3.0:1788844896.217951:0:68607:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001a2ddf4c 00000400:00000010:3.0:1788844896.217953:0:68607:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000001a2ddf4c. 00000400:00000200:3.0:1788844896.217956:0:68607:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:3.0:1788844896.217958:0:68607:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000045b70078. 00000400:00000010:3.0:1788844896.217961:0:68607:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000e6d51ddb. 00000001:02000400:1.0:1788844902.809991:0:69569:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 211: Remote NI recovery checks ======= 01:21:41 (1788844901) 00000001:02000400:0.0:1788844902.909159:0:69576:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure 00000400:00000200:1.0:1788844902.919472:0:69578:0:(net_fault.c:254:lnet_drop_rule_del()) src dst 00000400:00000200:0.0:1788844902.927193:0:68598:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: -108 00000400:00000200:0.0:1788844902.927212:0:68598:0:(peer.c:4385:lnet_peer_discovery()) stopping 00000400:00000200:0.0:1788844902.927225:0:68598:0:(peer.c:4431:lnet_peer_discovery()) stopped 00000400:00000200:1.0:1788844902.927419:0:69578:0:(peer.c:4488:lnet_peer_discovery_stop()) discovery stopped 00000400:00000010:1.0:1788844902.927604:0:69578:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 0000000072429e70. 00000400:00000010:1.0:1788844902.927611:0:69578:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 00000000dfde687d. 00000400:00000010:1.0:1788844902.927613:0:69578:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 00000000fcdf6c1f. 00000400:00000010:1.0:1788844902.927616:0:69578:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 000000008cfc270a. 00000400:00000010:1.0:1788844902.927618:0:69578:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 000000001ee9e81f. 00000400:00000010:1.0:1788844902.927619:0:69578:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 00000000fa541c26. 00000400:00000010:1.0:1788844902.927621:0:69578:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 000000000070272c. 00000400:00000010:1.0:1788844902.927624:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000001eee6f64. 00000400:00000010:1.0:1788844902.927627:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000f11e7c3e. 00000400:00000010:1.0:1788844902.927635:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000d8d54585. 00000400:00000010:1.0:1788844902.927637:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002780725a. 00000400:00000010:1.0:1788844902.927643:0:69578:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000a495e614. 00000400:00000200:1.0:1788844902.927646:0:69578:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000097e06605 00000400:00000010:1.0:1788844902.927649:0:69578:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 0000000097e06605. 00000400:00000200:1.0:1788844902.927653:0:69578:0:(api-ni.c:2393:lnet_push_target_event_handler()) type 6 status 0 unlinked 1 00000400:00000010:1.0:1788844902.927656:0:69578:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000024ee44a7. 00000400:00000400:1.0:1788844902.927662:0:69578:0:(lib-ptl.c:964:lnet_clear_lazy_portal()) Active lazy portal 0 on exit 00000400:00000010:1.0:1788844902.936089:0:69578:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000007d2552db. 00000400:00000200:1.0:1788844902.936098:0:69578:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000000e77ec59 00000400:00000010:1.0:1788844902.936100:0:69578:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000000e77ec59. 00000400:00000010:1.0:1788844902.936720:0:69578:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 73 at 00000000417ceea3. 00000400:00000200:0.0:1788844902.937222:0:68610:0:(acceptor.c:659:lnet_acceptor()) Acceptor stopping 00000400:00000200:1.0:1788844902.937382:0:69578:0:(router.c:792:lnet_del_route()) Del route: net <255:65535> : gw 00000400:00000200:1.0:1788844902.937403:0:69578:0:(lo.c:49:lolnd_shutdown()) shutdown 00000400:00000010:1.0:1788844902.937405:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000068c16029. 00000400:00000010:1.0:1788844902.937409:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000dbcf86c9. 00000400:00000010:1.0:1788844902.937411:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000070e563cb. 00000400:00000010:1.0:1788844902.937418:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000a89d7f34. 00000400:00000010:1.0:1788844902.937421:0:69578:0:(config.c:293:lnet_ni_free()) kfreed 'ni': 560 at 000000005fc77b98. 00000400:00000010:1.0:1788844902.937429:0:69578:0:(config.c:329:lnet_net_free()) kfreed 'net': 176 at 00000000641dd1e2. 00000800:00000200:1.1F:1788844902.937441:0:69578:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 2, ksnr_connected a ksnr_max_conns 1 00000800:00000200:1.1:1788844902.937457:0:69578:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 3, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:1.1:1788844902.937459:0:69578:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:1.1:1788844902.937460:0:69578:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 1, ksnr_connected 0 ksnr_max_conns 1 00000800:00000010:1.1:1788844902.937462:0:69578:0:(socklnd.c:126:ksocknal_destroy_conn_cb()) kfreed 'conn_cb': 192 at 00000000aee47713. 00000400:00020000:1.0:1788844902.937466:0:69578:0:(acceptor.c:252:lnet_acceptor_remove_socket()) Interface ens2 not found 00000800:00000200:2.0:1788844902.937926:0:68606:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 00000000a5ec57bb 00000800:00000010:2.0:1788844902.937931:0:68606:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 00000000a5ec57bb. 00000800:00000200:2.0:1788844902.937936:0:68606:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 00000000b94f5a0a 00000800:00000010:2.0:1788844902.937937:0:68606:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 00000000b94f5a0a. 00000800:00000200:2.0:1788844902.937961:0:68606:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 000000006ade1324 00000800:00000010:2.0:1788844902.937963:0:68606:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 000000006ade1324. 00000800:00000200:2.0:1788844902.937967:0:68606:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 00000000595b4ee4 00000800:00000200:2.0:1788844902.937973:0:68606:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.202.29@tcp 00000000a28c4abc deleted 00000800:00000010:2.0:1788844902.937975:0:68606:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 00000000a28c4abc. 00000800:00000010:2.0:1788844902.937977:0:68606:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 00000000595b4ee4. 00000800:00000010:1.0:1788844902.954087:0:69578:0:(socklnd.c:2498:ksocknal_shutdown()) kfreed 'net': 208 at 0000000089ec1266. 00000400:02000000:1.0:1788844902.954104:0:69578:0:(api-ni.c:2563:lnet_clear_zombies_nis_locked()) Removed LNI 192.168.202.29@tcp 00000400:00000010:1.0:1788844902.956649:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f3ada997. 00000400:00000010:1.0:1788844902.956658:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000677873ec. 00000400:00000010:1.0:1788844902.956660:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000000cc4a9c2. 00000400:00000010:1.0:1788844902.956661:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002d37fe28. 00000400:00000010:1.0:1788844902.956663:0:69578:0:(config.c:286:lnet_ni_free()) kfreed 'ni->ni_interface': 5 at 00000000e25e3788. 00000400:00000010:1.0:1788844902.956665:0:69578:0:(config.c:293:lnet_ni_free()) kfreed 'ni': 560 at 00000000abc33b89. 00000400:00000200:1.0:1788844902.956678:0:69578:0:(peer.c:620:lnet_peer_ni_clr_local_net_locked()) 000000000ac60c2a nid 192.168.202.29@tcp detached from local net 00000400:00000010:2.0:1788844902.957110:0:69578:0:(config.c:329:lnet_net_free()) kfreed 'net': 176 at 0000000016f26155. 00000400:00020000:2.0:1788844902.957133:0:69578:0:(acceptor.c:252:lnet_acceptor_remove_socket()) Interface ens2 not found 00000800:00000010:2.0:1788844902.957135:0:69578:0:(socklnd.c:2498:ksocknal_shutdown()) kfreed 'net': 208 at 0000000006ec8db9. 00000800:00000010:2.0:1788844902.957138:0:69578:0:(socklnd.c:2235:ksocknal_base_shutdown()) before NAL cleanup: kmem 418797 00000800:00000200:2.0:1788844902.957145:0:69578:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_GOING_DOWN, iface=lo ifindex 1 state 0 ns 000000004dc0f701 00000800:00000200:2.0:1788844902.957147:0:69578:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_DOWN, iface=lo ifindex 1 state 0 ns 000000004dc0f701 00000800:00000200:2.0:1788844902.957149:0:69578:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UNREGISTER, iface=lo ifindex 1 state 0 ns 000000004dc0f701 00000800:00000200:2.0:1788844902.957150:0:69578:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_GOING_DOWN, iface=ens2 ifindex 2 state 6 ns 000000004dc0f701 00000800:00000200:2.0:1788844902.957151:0:69578:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_DOWN, iface=ens2 ifindex 2 state 6 ns 000000004dc0f701 00000800:00000200:2.0:1788844902.957152:0:69578:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UNREGISTER, iface=ens2 ifindex 2 state 6 ns 000000004dc0f701 00000400:00000010:2.0:1788844902.972698:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 000000009c3a85f9. 00000400:00000010:2.0:1788844902.972707:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000c1b485da. 00000800:00000010:2.0:1788844902.972981:0:69578:0:(socklnd.c:2296:ksocknal_base_shutdown()) after NAL cleanup: kmem 418645 00000400:02000000:2.0:1788844902.972990:0:69578:0:(api-ni.c:2563:lnet_clear_zombies_nis_locked()) Removed LNI 192.168.202.29@tcp1 00000400:00000010:2.0:1788844902.980899:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000bf33960d. 00000400:00000010:2.0:1788844902.980913:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000018b7cfe4. 00000400:00000010:2.0:1788844902.980915:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000dd57f84e. 00000400:00000010:2.0:1788844902.980916:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000d63cdfc2. 00000400:00000010:2.0:1788844902.980918:0:69578:0:(config.c:286:lnet_ni_free()) kfreed 'ni->ni_interface': 5 at 00000000d384d4c6. 00000400:00000010:2.0:1788844902.980920:0:69578:0:(config.c:293:lnet_ni_free()) kfreed 'ni': 560 at 00000000d30daf99. 00000400:00000200:2.0:1788844902.980935:0:69578:0:(peer.c:442:lnet_peer_del_locked()) peer 192.168.202.29@tcp1 00000400:00000200:2.0:1788844902.980938:0:69578:0:(peer.c:364:lnet_peer_detach_peer_ni_locked()) peer 192.168.202.29@tcp1 NID 192.168.202.29@tcp 00000400:00000200:2.0:1788844902.980940:0:69578:0:(peer.c:2392:lnet_destroy_peer_ni_locked()) 000000000ac60c2a nid 192.168.202.29@tcp 00000400:00000010:2.0:1788844902.980942:0:69578:0:(peer.c:2422:lnet_destroy_peer_ni_locked()) kfreed 'lpni': 480 at 000000000ac60c2a. 00000400:00000200:2.0:1788844902.980944:0:69578:0:(peer.c:213:lnet_destroy_peer_net_locked()) 0000000024385f93 net tcp 00000400:00000010:2.0:1788844902.980946:0:69578:0:(peer.c:220:lnet_destroy_peer_net_locked()) kfreed 'lpn': 72 at 0000000024385f93. 00000400:00000200:2.0:1788844902.980948:0:69578:0:(peer.c:364:lnet_peer_detach_peer_ni_locked()) peer 192.168.202.29@tcp1 NID 192.168.202.29@tcp1 00000400:00000200:2.0:1788844902.980950:0:69578:0:(peer.c:2392:lnet_destroy_peer_ni_locked()) 00000000f4e70950 nid 192.168.202.29@tcp1 00000400:00000010:2.0:1788844902.980951:0:69578:0:(peer.c:2422:lnet_destroy_peer_ni_locked()) kfreed 'lpni': 480 at 00000000f4e70950. 00000400:00000200:2.0:1788844902.980953:0:69578:0:(peer.c:213:lnet_destroy_peer_net_locked()) 00000000fed799a0 net tcp1 00000400:00000010:2.0:1788844902.980954:0:69578:0:(peer.c:220:lnet_destroy_peer_net_locked()) kfreed 'lpn': 72 at 00000000fed799a0. 00000400:00000200:2.0:1788844902.980957:0:69578:0:(peer.c:274:lnet_destroy_peer_locked()) 000000008c1797eb nid 192.168.202.29@tcp1 00000400:00000010:2.0:1788844902.980959:0:69578:0:(peer.c:304:lnet_destroy_peer_locked()) kfreed 'lp': 360 at 000000008c1797eb. 00000400:00000010:2.0:1788844902.980962:0:69578:0:(config.c:329:lnet_net_free()) kfreed 'net': 176 at 000000004830a7e6. 00000400:00000010:2.0:1788844902.980966:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000a4a28bc5. 00000400:00000010:2.0:1788844902.980967:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000008ccbb23c. 00000400:00000010:2.0:1788844902.980972:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000062b2d79e. 00000400:00000010:2.0:1788844902.980976:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000006312873b. 00000400:00000010:2.0:1788844902.980977:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000008a3cf2bb. 00000400:00000010:2.0:1788844902.980978:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000888a09bb. 00000400:00000010:2.0:1788844902.980982:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000004950b76d. 00000400:00000010:2.0:1788844902.980985:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000059acbdd4. 00000400:00000010:2.0:1788844902.980986:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000002691533. 00000400:00000010:2.0:1788844902.980993:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000101b3909. 00000400:00000010:2.0:1788844902.980996:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000002ca18eb7. 00000400:00000010:2.0:1788844902.980997:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000005836113b. 00000400:00000010:2.0:1788844902.980998:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000493db505. 00000400:00000010:2.0:1788844902.980999:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000002e9f5976. 00000400:00000010:2.0:1788844902.981002:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000d7072bc3. 00000400:00000010:2.0:1788844902.981005:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000a7f7befd. 00000400:00000010:2.0:1788844902.981006:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000045514389. 00000400:00000010:2.0:1788844902.981006:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e99f0f74. 00000400:00000010:2.0:1788844902.981009:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000d584d457. 00000400:00000010:2.0:1788844902.981010:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000002b080a5b. 00000400:00000010:2.0:1788844902.981011:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000baf33e5d. 00000400:00000010:2.0:1788844902.981012:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000ad4dd87a. 00000400:00000010:2.0:1788844902.981015:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000dc206ed9. 00000400:00000010:2.0:1788844902.981017:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000039cdd270. 00000400:00000010:2.0:1788844902.981018:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000a9db2fb4. 00000400:00000010:2.0:1788844902.981019:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000028740d62. 00000400:00000010:2.0:1788844902.981021:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000b555330b. 00000400:00000010:2.0:1788844902.981023:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000468d16ad. 00000400:00000010:2.0:1788844902.981024:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000023af65e. 00000400:00000010:2.0:1788844902.981025:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e85f0291. 00000400:00000010:2.0:1788844902.981027:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000004c570512. 00000400:00000010:2.0:1788844902.981029:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000d447cea9. 00000400:00000010:2.0:1788844902.981030:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000d24b9af7. 00000400:00000010:2.0:1788844902.981031:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000006e31b303. 00000400:00000010:2.0:1788844902.981033:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000049f837f7. 00000400:00000010:2.0:1788844902.981034:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000001db122b3. 00000400:00000010:2.0:1788844902.981035:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000ba407268. 00000400:00000010:2.0:1788844902.981036:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000ac6e27cf. 00000400:00000010:2.0:1788844902.981038:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000003c7946d5. 00000400:00000010:2.0:1788844902.981040:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000003f207307. 00000400:00000010:2.0:1788844902.981040:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000e240750e. 00000400:00000010:2.0:1788844902.981041:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000062556f66. 00000400:00000010:2.0:1788844902.981047:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000006051c759. 00000400:00000010:2.0:1788844902.981048:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000003ad6ea45. 00000400:00000010:2.0:1788844902.981049:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000c2f823b2. 00000400:00000010:2.0:1788844902.981050:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000bc34c189. 00000400:00000010:2.0:1788844902.981053:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000e0fb5a6a. 00000400:00000010:2.0:1788844902.981054:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000305212d9. 00000400:00000010:2.0:1788844902.981055:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000028083766. 00000400:00000010:2.0:1788844902.981056:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000a9800616. 00000400:00000010:2.0:1788844902.981059:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000004e1cda6e. 00000400:00000010:2.0:1788844902.981061:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000072be2e75. 00000400:00000010:2.0:1788844902.981062:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000eed24453. 00000400:00000010:2.0:1788844902.981063:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000026ed2ce0. 00000400:00000010:2.0:1788844902.981065:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000e54e880c. 00000400:00000010:2.0:1788844902.981067:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000ee49a149. 00000400:00000010:2.0:1788844902.981069:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000004b77f11c. 00000400:00000010:2.0:1788844902.981070:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000002b712343. 00000400:00000010:2.0:1788844902.981074:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000046f7e0d3. 00000400:00000010:2.0:1788844902.981076:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000001d9bb8c8. 00000400:00000010:2.0:1788844902.981077:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000363a54ae. 00000400:00000010:2.0:1788844902.981078:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000581ae48b. 00000400:00000010:2.0:1788844902.981080:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f830d99c. 00000400:00000010:2.0:1788844902.981082:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000000e5d5b49. 00000400:00000010:2.0:1788844902.981082:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000005f6e227b. 00000400:00000010:2.0:1788844902.981083:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000200e3a59. 00000400:00000010:2.0:1788844902.981086:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000fcd20a28. 00000400:00000010:2.0:1788844902.981088:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000007e0d620. 00000400:00000010:2.0:1788844902.981088:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000004f96d58c. 00000400:00000010:2.0:1788844902.981089:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000835f7a68. 00000400:00000010:2.0:1788844902.981091:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000000b186f96. 00000400:00000010:2.0:1788844902.981093:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000040cdf5ec. 00000400:00000010:2.0:1788844902.981094:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000006591c073. 00000400:00000010:2.0:1788844902.981095:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000a9c9fd57. 00000400:00000010:2.0:1788844902.981097:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000732bddc8. 00000400:00000010:2.0:1788844902.981103:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000bc61d396. 00000400:00000010:2.0:1788844902.981104:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000098462bc. 00000400:00000010:2.0:1788844902.981106:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000006764a14e. 00000400:00000010:2.0:1788844902.981109:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f8fd273d. 00000400:00000010:2.0:1788844902.981111:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000009b68a820. 00000400:00000010:2.0:1788844902.981112:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000b308ad10. 00000400:00000010:2.0:1788844902.981112:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000060e4fd94. 00000400:00000010:2.0:1788844902.981115:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000004c3f562e. 00000400:00000010:2.0:1788844902.981117:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000006898b918. 00000400:00000010:2.0:1788844902.981118:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000cf697eee. 00000400:00000010:2.0:1788844902.981119:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000072f5f159. 00000400:00000010:2.0:1788844902.981121:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000002383e933. 00000400:00000010:2.0:1788844902.981123:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000009a63222b. 00000400:00000010:2.0:1788844902.981123:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000741f4313. 00000400:00000010:2.0:1788844902.981124:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000f9964600. 00000400:00000010:2.0:1788844902.981127:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000052a76679. 00000400:00000010:2.0:1788844902.981128:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000091848fd4. 00000400:00000010:2.0:1788844902.981129:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000f355510d. 00000400:00000010:2.0:1788844902.981130:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000007966fc70. 00000400:00000010:2.0:1788844902.981132:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000071fab5be. 00000400:00000010:2.0:1788844902.981147:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000018f4d00d. 00000400:00000010:2.0:1788844902.981148:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000ee8dfa2e. 00000400:00000010:2.0:1788844902.981149:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000009b36ed2c. 00000400:00000010:2.0:1788844902.981152:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000aa6f72a4. 00000400:00000010:2.0:1788844902.981153:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000ef1655be. 00000400:00000010:2.0:1788844902.981154:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000009116a73c. 00000400:00000010:2.0:1788844902.981155:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000b8a50ccf. 00000400:00000010:2.0:1788844902.981158:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000048876783. 00000400:00000010:2.0:1788844902.981159:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000019f8f883. 00000400:00000010:2.0:1788844902.981160:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000fb2d6842. 00000400:00000010:2.0:1788844902.981161:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000000a6d6962. 00000400:00000010:2.0:1788844902.981163:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000beeb68e0. 00000400:00000010:2.0:1788844902.981165:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000005a7f0f8f. 00000400:00000010:2.0:1788844902.981169:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000161d5ed6. 00000400:00000010:2.0:1788844902.981170:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000003bc85b5d. 00000400:00000010:2.0:1788844902.981173:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000005aefee44. 00000400:00000010:2.0:1788844902.981175:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000c4c8bad6. 00000400:00000010:2.0:1788844902.981176:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000072acab06. 00000400:00000010:2.0:1788844902.981177:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000092971a5c. 00000400:00000010:2.0:1788844902.981180:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000035c7b992. 00000400:00000010:2.0:1788844902.981182:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f66e8904. 00000400:00000010:2.0:1788844902.981183:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000003c313220. 00000400:00000010:2.0:1788844902.981184:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000062fe75ae. 00000400:00000010:2.0:1788844902.981186:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000c587f8d0. 00000400:00000010:2.0:1788844902.981191:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000003a5177eb. 00000400:00000010:2.0:1788844902.981192:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000007060eeb1. 00000400:00000010:2.0:1788844902.981194:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000814c4bb6. 00000400:00000010:2.0:1788844902.981196:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000004b3231fa. 00000400:00000010:2.0:1788844902.981198:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000c64cc704. 00000400:00000010:2.0:1788844902.981198:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000007d1d0893. 00000400:00000010:2.0:1788844902.981200:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000005a95a6de. 00000400:00000010:2.0:1788844902.981202:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000058d5f192. 00000400:00000010:2.0:1788844902.981203:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000009b2f5c5. 00000400:00000010:2.0:1788844902.981204:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002577d682. 00000400:00000010:2.0:1788844902.981205:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000dc729434. 00000400:00000010:2.0:1788844902.981208:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000b7b45c03. 00000400:00000010:2.0:1788844902.981209:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000a30ddda9. 00000400:00000010:2.0:1788844902.981210:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000feebb32e. 00000400:00000010:2.0:1788844902.981210:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000006c09f49. 00000400:00000010:2.0:1788844902.981213:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000012bb5fad. 00000400:00000010:2.0:1788844902.981215:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000b83ed0df. 00000400:00000010:2.0:1788844902.981216:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000012f5b268. 00000400:00000010:2.0:1788844902.981217:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000a39c78b7. 00000400:00000010:2.0:1788844902.981219:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000008608d7f3. 00000400:00000010:2.0:1788844902.981221:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000002c777628. 00000400:00000010:2.0:1788844902.981222:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002f9882bf. 00000400:00000010:2.0:1788844902.981228:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000007178c36a. 00000400:00000010:2.0:1788844902.981230:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000001c3d3225. 00000400:00000010:2.0:1788844902.981234:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000fc2d389e. 00000400:00000010:2.0:1788844902.981235:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000044352509. 00000400:00000010:2.0:1788844902.981236:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000f93ac11e. 00000400:00000010:2.0:1788844902.981238:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000078e45bfe. 00000400:00000010:2.0:1788844902.981240:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000cbe9af0b. 00000400:00000010:2.0:1788844902.981241:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000014991cc7. 00000400:00000010:2.0:1788844902.981241:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000db66b092. 00000400:00000010:2.0:1788844902.981244:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000073a778bd. 00000400:00000010:2.0:1788844902.981247:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000503c1090. 00000400:00000010:2.0:1788844902.981248:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000bb786270. 00000400:00000010:2.0:1788844902.981249:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000014da4132. 00000400:00000010:2.0:1788844902.981251:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000001c0ab122. 00000400:00000010:2.0:1788844902.981257:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000005c24ceee. 00000400:00000010:2.0:1788844902.981258:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002551acd8. 00000400:00000010:2.0:1788844902.981259:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000235025b8. 00000400:00000010:2.0:1788844902.981261:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000cce5a98d. 00000400:00000010:2.0:1788844902.981264:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000000ee4022c. 00000400:00000010:2.0:1788844902.981265:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000ab70581a. 00000400:00000010:2.0:1788844902.981267:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000c7581a2f. 00000400:00000010:2.0:1788844902.981270:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000b06228b2. 00000400:00000010:2.0:1788844902.981272:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000029b104f2. 00000400:00000010:2.0:1788844902.981274:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000e964c8d9. 00000400:00000010:2.0:1788844902.981276:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000bfd6027d. 00000400:00000010:2.0:1788844902.981279:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000011c2bc27. 00000400:00000010:2.0:1788844902.981282:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000322b02b6. 00000400:00000010:2.0:1788844902.981284:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000000f30b4f1. 00000400:00000010:2.0:1788844902.981286:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000006fa59e70. 00000400:00000010:2.0:1788844902.981291:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000090d28e3a. 00000400:00000010:2.0:1788844902.981295:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000003efbdae6. 00000400:00000010:2.0:1788844902.981298:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000000140882c. 00000400:00000010:2.0:1788844902.981301:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000d8145822. 00000400:00000010:2.0:1788844902.981310:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000b6393d4e. 00000400:00000010:2.0:1788844902.981312:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000060204fe1. 00000400:00000010:2.0:1788844902.981314:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000091e2ad38. 00000400:00000010:2.0:1788844902.981316:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000a9becf6e. 00000400:00000010:2.0:1788844902.981320:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000004d403923. 00000400:00000010:2.0:1788844902.981322:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000005ac03011. 00000400:00000010:2.0:1788844902.981324:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000080d46809. 00000400:00000010:2.0:1788844902.981325:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000979c21a9. 00000400:00000010:2.0:1788844902.981329:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000094e9313b. 00000400:00000010:2.0:1788844902.981331:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000082d1b873. 00000400:00000010:2.0:1788844902.981333:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000050551255. 00000400:00000010:2.0:1788844902.981334:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000007dbb7b21. 00000400:00000010:2.0:1788844902.981338:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000e492d794. 00000400:00000010:2.0:1788844902.981341:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000007c668254. 00000400:00000010:2.0:1788844902.981342:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000004bc073f9. 00000400:00000010:2.0:1788844902.981343:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000353dad28. 00000400:00000010:2.0:1788844902.981345:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ea9620cd. 00000400:00000010:2.0:1788844902.981346:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000005c7b5d24. 00000400:00000010:2.0:1788844902.981347:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000009d92ba7b. 00000400:00000010:2.0:1788844902.981348:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000001b0e9840. 00000400:00000010:2.0:1788844902.981351:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f4841a6c. 00000400:00000010:2.0:1788844902.981352:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000e9443202. 00000400:00000010:2.0:1788844902.981353:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002226e38c. 00000400:00000010:2.0:1788844902.981354:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000024a8ae36. 00000400:00000010:2.0:1788844902.981356:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000e29cd06f. 00000400:00000010:2.0:1788844902.981357:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000002631df0d. 00000400:00000010:2.0:1788844902.981358:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000fcbb744a. 00000400:00000010:2.0:1788844902.981359:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000a711f7ba. 00000400:00000010:2.0:1788844902.981361:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ce23c6c8. 00000400:00000010:2.0:1788844902.981371:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000000621c622. 00000400:00000010:2.0:1788844902.981373:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000083786cbf. 00000400:00000010:2.0:1788844902.981374:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000076460c26. 00000400:00000010:2.0:1788844902.981377:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000008f2d5864. 00000400:00000010:2.0:1788844902.981381:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000de099ecc. 00000400:00000010:2.0:1788844902.981382:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000dde3929c. 00000400:00000010:2.0:1788844902.981383:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000004500d76a. 00000400:00000010:2.0:1788844902.981386:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000005d51b27f. 00000400:00000010:2.0:1788844902.981387:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000326701d8. 00000400:00000010:2.0:1788844902.981388:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000001681843c. 00000400:00000010:2.0:1788844902.981388:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000004f1bb23c. 00000400:00000010:2.0:1788844902.981391:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000148b82bf. 00000400:00000010:2.0:1788844902.981392:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000003cc657d3. 00000400:00000010:2.0:1788844902.981393:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000162133db. 00000400:00000010:2.0:1788844902.981394:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000b5291ce4. 00000400:00000010:2.0:1788844902.981396:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000025943f65. 00000400:00000010:2.0:1788844902.981398:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000000f199da0. 00000400:00000010:2.0:1788844902.981399:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000e329392f. 00000400:00000010:2.0:1788844902.981400:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000002e43d554. 00000400:00000010:2.0:1788844902.981402:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000007e760ea. 00000400:00000010:2.0:1788844902.981403:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f60f4112. 00000400:00000010:2.0:1788844902.981404:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000072c72039. 00000400:00000010:2.0:1788844902.981404:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000aa90fcc5. 00000400:00000010:2.0:1788844902.981407:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000535203ad. 00000400:00000010:2.0:1788844902.981408:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000099de7f65. 00000400:00000010:2.0:1788844902.981409:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000a60a864c. 00000400:00000010:2.0:1788844902.981410:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000f53e5427. 00000400:00000010:2.0:1788844902.981412:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000099676174. 00000400:00000010:2.0:1788844902.981414:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000b33e6f99. 00000400:00000010:2.0:1788844902.981415:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000005c6b28d3. 00000400:00000010:2.0:1788844902.981416:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000004ccc80bf. 00000400:00000010:2.0:1788844902.981418:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000006b982f8a. 00000400:00000010:2.0:1788844902.981420:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000757bc96f. 00000400:00000010:2.0:1788844902.981420:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000a8e07715. 00000400:00000010:2.0:1788844902.981421:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000019ef2462. 00000400:00000010:2.0:1788844902.981424:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000cf800bcd. 00000400:00000010:2.0:1788844902.981425:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000d3850584. 00000400:00000010:2.0:1788844902.981429:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000da79684b. 00000400:00000010:2.0:1788844902.981430:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000003aa18ec1. 00000400:00000010:2.0:1788844902.981432:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000005b44c76a. 00000400:00000010:2.0:1788844902.981433:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000836770ac. 00000400:00000010:2.0:1788844902.981434:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000005822650b. 00000400:00000010:2.0:1788844902.981435:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000b8a26942. 00000400:00000010:2.0:1788844902.981437:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000000c0d76c6. 00000400:00000010:2.0:1788844902.981439:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000188fd018. 00000400:00000010:2.0:1788844902.981440:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000093d98ddc. 00000400:00000010:2.0:1788844902.981440:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000d5f64747. 00000400:00000010:2.0:1788844902.981443:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000008feb9acd. 00000400:00000010:2.0:1788844902.981444:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000700da446. 00000400:00000010:2.0:1788844902.981445:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002f9f85de. 00000400:00000010:2.0:1788844902.981446:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000c517f7e5. 00000400:00000010:2.0:1788844902.981449:0:69578:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000df507ca1. 00000400:00000010:2.0:1788844902.981450:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000fa5f2aa8. 00000400:00000010:2.0:1788844902.981451:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002210c718. 00000400:00000010:2.0:1788844902.981452:0:69578:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000016024adb. 00000400:00000010:2.0:1788844902.981453:0:69578:0:(lib-ptl.c:854:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals': 512 at 00000000aff1e4cf. 00000400:00000010:2.0:1788844902.981456:0:69578:0:(api-ni.c:1205:lnet_res_container_cleanup()) kfreed 'rec->rec_lh_hash': 65536 at 00000000e3176fe9. 00000400:00000010:2.0:1788844902.981469:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000001772a699. 00000400:00000010:2.0:1788844902.981470:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000004f388c65. 00000400:00000010:2.0:1788844902.981471:0:69578:0:(api-ni.c:1205:lnet_res_container_cleanup()) kfreed 'rec->rec_lh_hash': 65536 at 000000005572738d. 00000400:00000010:2.0:1788844902.981473:0:69578:0:(lib-msg.c:1308:lnet_msg_container_cleanup()) kfreed 'container->msc_finalizers': 32 at 00000000b7ef994e. 00000400:00000010:2.0:1788844902.981475:0:69578:0:(lib-msg.c:1314:lnet_msg_container_cleanup()) kfreed 'container->msc_resenders': 32 at 00000000d85ba17b. 00000400:00000010:2.0:1788844902.981476:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 000000000b454fab. 00000400:00000010:2.0:1788844902.981477:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000c45e19d2. 00000400:00000010:2.0:1788844902.981481:0:69578:0:(peer.c:85:lnet_peer_tables_destroy()) kfreed 'hash': 8192 at 0000000003ba3b00. 00000400:00000010:2.0:1788844902.981488:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 000000008e269c04. 00000400:00000010:2.0:1788844902.981491:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000c04d4d2a. 00000400:00000010:2.0:1788844902.981493:0:69578:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 000000007a2ca541. 00000400:00000010:2.0:1788844902.981494:0:69578:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000044e89cc2. 00000400:00000010:2.0:1788844902.981499:0:69578:0:(api-ni.c:700:lnet_destroy_remote_nets_table()) kfreed 'the_lnet.ln_remote_nets_hash': 1024 at 000000000956d444. 00000001:00000200:2.0:1788844902.981502:0:69578:0:(udsp.c:1126:lnet_udsp_destroy()) Destroying UDSPs in the system 00000001:02000400:2.0:1788844903.016445:0:69585:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure 00000400:00000010:3.0:1788844903.050937:0:69586:0:(api-ni.c:676:lnet_create_remote_nets_table()) alloc '(hash)': 1024 at 00000000f7f65837. 00000400:00000010:3.0:1788844903.050948:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000006e8e4304. 00000400:00000010:3.0:1788844903.050953:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000c33ab6cb. 00000400:00000010:3.0:1788844903.050956:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000024eb8ae2. 00000400:00000010:3.0:1788844903.050965:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000ffde6bcd. 00000400:00000010:3.0:1788844903.050970:0:69586:0:(peer.c:109:lnet_peer_tables_create()) alloc '(hash)': 8192 at 000000004d481902. 00000400:00000010:3.0:1788844903.050975:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000962292a7. 00000400:00000010:3.0:1788844903.050977:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000098e8f8c. 00000400:00000010:3.0:1788844903.050981:0:69586:0:(lib-msg.c:1338:lnet_msg_container_setup()) alloc '(container->msc_finalizers)': 32 at 0000000033014b15. 00000400:00000010:3.0:1788844903.050983:0:69586:0:(lib-msg.c:1348:lnet_msg_container_setup()) alloc '(container->msc_resenders)': 32 at 00000000c244c076. 00000400:00000010:3.0:1788844903.051055:0:69586:0:(api-ni.c:1227:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 00000000b244745d. 00000400:00000010:3.0:1788844903.051074:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000109af81f. 00000400:00000010:3.0:1788844903.051077:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b083c074. 00000400:00000010:3.0:1788844903.051135:0:69586:0:(api-ni.c:1227:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 00000000ffacfe34. 00000400:00000010:3.0:1788844903.051154:0:69586:0:(lib-ptl.c:864:lnet_portals_create()) alloc '(the_lnet.ln_portals)': 512 at 000000008d81bb62. 00000400:00000010:3.0:1788844903.051158:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000065731974. 00000400:00000010:3.0:1788844903.051160:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000406ca20f. 00000400:00000010:3.0:1788844903.051162:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fc0cfe08. 00000400:00000010:3.0:1788844903.051167:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005636bcd6. 00000400:00000010:3.0:1788844903.051170:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000004c8a6973. 00000400:00000010:3.0:1788844903.051172:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000031b7289f. 00000400:00000010:3.0:1788844903.051174:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003f664474. 00000400:00000010:3.0:1788844903.051177:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000595b4ee4. 00000400:00000010:3.0:1788844903.051180:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000004096d1cf. 00000400:00000010:3.0:1788844903.051182:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000000de8427e. 00000400:00000010:3.0:1788844903.051183:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c984575c. 00000400:00000010:3.0:1788844903.051187:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000019bdd1d9. 00000400:00000010:3.0:1788844903.051191:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000fed799a0. 00000400:00000010:3.0:1788844903.051193:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d9819b09. 00000400:00000010:3.0:1788844903.051195:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009a5490b5. 00000400:00000010:3.0:1788844903.051199:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003abcc076. 00000400:00000010:3.0:1788844903.051203:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000067c24a58. 00000400:00000010:3.0:1788844903.051205:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e895244b. 00000400:00000010:3.0:1788844903.051207:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a60edc7f. 00000400:00000010:3.0:1788844903.051211:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000cce1b85e. 00000400:00000010:3.0:1788844903.051213:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e14976b5. 00000400:00000010:3.0:1788844903.051221:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000060c544f. 00000400:00000010:3.0:1788844903.051223:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000083415f8f. 00000400:00000010:3.0:1788844903.051227:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f8b96da2. 00000400:00000010:3.0:1788844903.051231:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e0634bff. 00000400:00000010:3.0:1788844903.051232:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000fe1c54f1. 00000400:00000010:3.0:1788844903.051234:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000818f8a99. 00000400:00000010:3.0:1788844903.051238:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002bbfaf7b. 00000400:00000010:3.0:1788844903.051241:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000009141af7. 00000400:00000010:3.0:1788844903.051243:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000009e80ec08. 00000400:00000010:3.0:1788844903.051245:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000bf7845de. 00000400:00000010:3.0:1788844903.051250:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001f03eb7b. 00000400:00000010:3.0:1788844903.051253:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000003bd6e968. 00000400:00000010:3.0:1788844903.051255:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000c45bae3a. 00000400:00000010:3.0:1788844903.051256:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005b48385f. 00000400:00000010:3.0:1788844903.051261:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f8577e7d. 00000400:00000010:3.0:1788844903.051265:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000bae90281. 00000400:00000010:3.0:1788844903.051267:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000796d72dc. 00000400:00000010:3.0:1788844903.051269:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000064ba31c8. 00000400:00000010:3.0:1788844903.051272:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000625a9453. 00000400:00000010:3.0:1788844903.051275:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ce84b06d. 00000400:00000010:3.0:1788844903.051276:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000007e128e28. 00000400:00000010:3.0:1788844903.051278:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000085b9359. 00000400:00000010:3.0:1788844903.051282:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000008a53d52. 00000400:00000010:3.0:1788844903.051285:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000aef65602. 00000400:00000010:3.0:1788844903.051287:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001da49bc8. 00000400:00000010:3.0:1788844903.051290:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000004d273e1. 00000400:00000010:3.0:1788844903.051296:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000dd0c6b59. 00000400:00000010:3.0:1788844903.051299:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000008c06e919. 00000400:00000010:3.0:1788844903.051301:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000faefeea4. 00000400:00000010:3.0:1788844903.051303:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ddbf54cc. 00000400:00000010:3.0:1788844903.051307:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000026a45d92. 00000400:00000010:3.0:1788844903.051310:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000085c4398b. 00000400:00000010:3.0:1788844903.051317:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000442dadfb. 00000400:00000010:3.0:1788844903.051320:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000003be6f71. 00000400:00000010:3.0:1788844903.051323:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000365db3e4. 00000400:00000010:3.0:1788844903.051326:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000007a8ccd8b. 00000400:00000010:3.0:1788844903.051328:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001616a2af. 00000400:00000010:3.0:1788844903.051330:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002779f1ce. 00000400:00000010:3.0:1788844903.051334:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ce632e44. 00000400:00000010:3.0:1788844903.051337:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000028431340. 00000400:00000010:3.0:1788844903.051338:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e61a5bde. 00000400:00000010:3.0:1788844903.051340:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a83e7e57. 00000400:00000010:3.0:1788844903.051345:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001dc6f449. 00000400:00000010:3.0:1788844903.051348:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000afab913f. 00000400:00000010:3.0:1788844903.051350:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000023abbb72. 00000400:00000010:3.0:1788844903.051351:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000248a6efb. 00000400:00000010:3.0:1788844903.051355:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f2896859. 00000400:00000010:3.0:1788844903.051358:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000049403b1d. 00000400:00000010:3.0:1788844903.051360:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000417090ac. 00000400:00000010:3.0:1788844903.051362:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000080acbd77. 00000400:00000010:3.0:1788844903.051366:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000666acba5. 00000400:00000010:3.0:1788844903.051368:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ba313706. 00000400:00000010:3.0:1788844903.051370:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005c86a724. 00000400:00000010:3.0:1788844903.051372:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005450bc93. 00000400:00000010:3.0:1788844903.051375:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000085d84fe0. 00000400:00000010:3.0:1788844903.051379:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000002f400dfa. 00000400:00000010:3.0:1788844903.051380:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000406cc200. 00000400:00000010:3.0:1788844903.051382:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000386ff875. 00000400:00000010:3.0:1788844903.051387:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ac6fdf8f. 00000400:00000010:3.0:1788844903.051389:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ca35aead. 00000400:00000010:3.0:1788844903.051391:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000add70cbc. 00000400:00000010:3.0:1788844903.051393:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e59582d5. 00000400:00000010:3.0:1788844903.051397:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b075f539. 00000400:00000010:3.0:1788844903.051399:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000016f2e852. 00000400:00000010:3.0:1788844903.051408:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000aea4744f. 00000400:00000010:3.0:1788844903.051410:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000005399dc9. 00000400:00000010:3.0:1788844903.051414:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000091cba7af. 00000400:00000010:3.0:1788844903.051417:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e627d6a9. 00000400:00000010:3.0:1788844903.051418:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d11d8cd3. 00000400:00000010:3.0:1788844903.051420:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000084a141da. 00000400:00000010:3.0:1788844903.051424:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000041fda1eb. 00000400:00000010:3.0:1788844903.051426:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e624a748. 00000400:00000010:3.0:1788844903.051428:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000003e9b1fb4. 00000400:00000010:3.0:1788844903.051430:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000bc812c2a. 00000400:00000010:3.0:1788844903.051435:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006051c759. 00000400:00000010:3.0:1788844903.051437:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000082c3d7ae. 00000400:00000010:3.0:1788844903.051439:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000003f54b82f. 00000400:00000010:3.0:1788844903.051441:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003a74fcc1. 00000400:00000010:3.0:1788844903.051445:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003c7946d5. 00000400:00000010:3.0:1788844903.051448:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000406afaf8. 00000400:00000010:3.0:1788844903.051449:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002f2618a5. 00000400:00000010:3.0:1788844903.051451:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008334e28c. 00000400:00000010:3.0:1788844903.051455:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000049f837f7. 00000400:00000010:3.0:1788844903.051457:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000fc8d4ff3. 00000400:00000010:3.0:1788844903.051459:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000fb507bf3. 00000400:00000010:3.0:1788844903.051461:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000217cf18f. 00000400:00000010:3.0:1788844903.051464:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004c570512. 00000400:00000010:3.0:1788844903.051467:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000624d8cba. 00000400:00000010:3.0:1788844903.051469:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000b6e0beba. 00000400:00000010:3.0:1788844903.051471:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000be5e9941. 00000400:00000010:3.0:1788844903.051475:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b555330b. 00000400:00000010:3.0:1788844903.051478:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000aa1008e9. 00000400:00000010:3.0:1788844903.051480:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001ecd7c69. 00000400:00000010:3.0:1788844903.051481:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000dedcd839. 00000400:00000010:3.0:1788844903.051485:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000dc206ed9. 00000400:00000010:3.0:1788844903.051487:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e64a6c1e. 00000400:00000010:3.0:1788844903.051495:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005bb316ba. 00000400:00000010:3.0:1788844903.051497:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007978d37a. 00000400:00000010:3.0:1788844903.051501:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d584d457. 00000400:00000010:3.0:1788844903.051504:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000004ef956bc. 00000400:00000010:3.0:1788844903.051506:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000086622aa7. 00000400:00000010:3.0:1788844903.051508:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000659cb8f1. 00000400:00000010:3.0:1788844903.051511:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d7072bc3. 00000400:00000010:3.0:1788844903.051514:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000071e56a46. 00000400:00000010:3.0:1788844903.051516:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000f556e4e2. 00000400:00000010:3.0:1788844903.051518:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000dfdd4354. 00000400:00000010:3.0:1788844903.051523:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000035c7b992. 00000400:00000010:3.0:1788844903.051543:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000009d22323b. 00000400:00000010:3.0:1788844903.051546:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000043dcb519. 00000400:00000010:3.0:1788844903.051548:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000302c289e. 00000400:00000010:3.0:1788844903.051553:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005aefee44. 00000400:00000010:3.0:1788844903.051556:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000b5fc2b56. 00000400:00000010:3.0:1788844903.051557:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ff961f49. 00000400:00000010:3.0:1788844903.051559:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008ed17bca. 00000400:00000010:3.0:1788844903.051564:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000012bb5fad. 00000400:00000010:3.0:1788844903.051566:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000064f0c89d. 00000400:00000010:3.0:1788844903.051568:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a47d9b88. 00000400:00000010:3.0:1788844903.051571:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000222cbdfd. 00000400:00000010:3.0:1788844903.051575:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b7b45c03. 00000400:00000010:3.0:1788844903.051578:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000006075400d. 00000400:00000010:3.0:1788844903.051580:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002f33c48b. 00000400:00000010:3.0:1788844903.051582:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000bd712de2. 00000400:00000010:3.0:1788844903.051586:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000058d5f192. 00000400:00000010:3.0:1788844903.051589:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000f1b77af6. 00000400:00000010:3.0:1788844903.051591:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000099b9809d. 00000400:00000010:3.0:1788844903.051593:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000020b3104b. 00000400:00000010:3.0:1788844903.051597:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004b3231fa. 00000400:00000010:3.0:1788844903.051600:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000052ffac95. 00000400:00000010:3.0:1788844903.051608:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001674e34b. 00000400:00000010:3.0:1788844903.051610:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001b227634. 00000400:00000010:3.0:1788844903.051614:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000c587f8d0. 00000400:00000010:3.0:1788844903.051616:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000000e13b884. 00000400:00000010:3.0:1788844903.051618:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000dbfb5833. 00000400:00000010:3.0:1788844903.051620:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000060e4d16. 00000400:00000010:3.0:1788844903.051635:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001ba26530. 00000400:00000010:3.0:1788844903.051640:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000006379be4f. 00000400:00000010:3.0:1788844903.051642:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000000202246d. 00000400:00000010:3.0:1788844903.051644:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f0e8c66a. 00000400:00000010:3.0:1788844903.051649:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d8f2cce8. 00000400:00000010:3.0:1788844903.051652:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000076532758. 00000400:00000010:3.0:1788844903.051654:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000fcb9842d. 00000400:00000010:3.0:1788844903.051656:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e2d09096. 00000400:00000010:3.0:1788844903.051661:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e5717462. 00000400:00000010:3.0:1788844903.051664:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000001dfc40c1. 00000400:00000010:3.0:1788844903.051666:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001d092066. 00000400:00000010:3.0:1788844903.051667:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008cfacad6. 00000400:00000010:3.0:1788844903.051673:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f0f3900e. 00000400:00000010:3.0:1788844903.051675:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000afa3651a. 00000400:00000010:3.0:1788844903.051677:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000015a7eedd. 00000400:00000010:3.0:1788844903.051679:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fa63a178. 00000400:00000010:3.0:1788844903.051689:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000003ba3b00. 00000400:00000010:3.0:1788844903.051693:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000034b85e67. 00000400:00000010:3.0:1788844903.051696:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000008e1d2ffa. 00000400:00000010:3.0:1788844903.051698:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e8cac664. 00000400:00000010:3.0:1788844903.051701:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000062b2d79e. 00000400:00000010:3.0:1788844903.051704:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000dc627611. 00000400:00000010:3.0:1788844903.051706:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a5a0fca5. 00000400:00000010:3.0:1788844903.051708:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e5b2fa44. 00000400:00000010:3.0:1788844903.051712:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004950b76d. 00000400:00000010:3.0:1788844903.051714:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000092363bc8. 00000400:00000010:3.0:1788844903.051721:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000bd38ab3a. 00000400:00000010:3.0:1788844903.051722:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009ef0139b. 00000400:00000010:3.0:1788844903.051726:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002ca18eb7. 00000400:00000010:3.0:1788844903.051729:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000046fce6dd. 00000400:00000010:3.0:1788844903.051731:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000003fe01660. 00000400:00000010:3.0:1788844903.051733:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002953d5ee. 00000400:00000010:3.0:1788844903.051744:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e492d794. 00000400:00000010:3.0:1788844903.051748:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000090b4a2cb. 00000400:00000010:3.0:1788844903.051749:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ba12b6e3. 00000400:00000010:3.0:1788844903.051752:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ff1cd0ef. 00000400:00000010:3.0:1788844903.051756:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ea9620cd. 00000400:00000010:3.0:1788844903.051759:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000003bd47c82. 00000400:00000010:3.0:1788844903.051761:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000022adf9dc. 00000400:00000010:3.0:1788844903.051762:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fe307cec. 00000400:00000010:3.0:1788844903.051768:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f4841a6c. 00000400:00000010:3.0:1788844903.051771:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e29ecc97. 00000400:00000010:3.0:1788844903.051773:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000233100cf. 00000400:00000010:3.0:1788844903.051775:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007f2c1e88. 00000400:00000010:3.0:1788844903.051804:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e29cd06f. 00000400:00000010:3.0:1788844903.051808:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000052a64fe4. 00000400:00000010:3.0:1788844903.051810:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d68be744. 00000400:00000010:3.0:1788844903.051812:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ed069f39. 00000400:00000010:3.0:1788844903.051824:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b6393d4e. 00000400:00000010:3.0:1788844903.051828:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000907bc77a. 00000400:00000010:3.0:1788844903.051830:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000005f64e2c. 00000400:00000010:3.0:1788844903.051832:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007b0c6120. 00000400:00000010:3.0:1788844903.051837:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004d403923. 00000400:00000010:3.0:1788844903.051840:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000292c3b3c. 00000400:00000010:3.0:1788844903.051842:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000fb207986. 00000400:00000010:3.0:1788844903.051844:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000b0cf052. 00000400:00000010:3.0:1788844903.051849:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000094e9313b. 00000400:00000010:3.0:1788844903.051852:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000007fb8397b. 00000400:00000010:3.0:1788844903.051858:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000009486a532. 00000400:00000010:3.0:1788844903.051859:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002d892f93. 00000400:00000010:3.0:1788844903.051865:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000090d28e3a. 00000400:00000010:3.0:1788844903.051868:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000f47d3b45. 00000400:00000010:3.0:1788844903.051870:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000013c3a0f2. 00000400:00000010:3.0:1788844903.051871:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000803935c1. 00000400:00000010:3.0:1788844903.051880:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b06228b2. 00000400:00000010:3.0:1788844903.051884:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000080a083d2. 00000400:00000010:3.0:1788844903.051885:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a2016c52. 00000400:00000010:3.0:1788844903.051887:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000614fdb83. 00000400:00000010:3.0:1788844903.051891:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000011c2bc27. 00000400:00000010:3.0:1788844903.051893:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000001775e3f2. 00000400:00000010:3.0:1788844903.051895:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002c3ab64d. 00000400:00000010:3.0:1788844903.051896:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b906f0cd. 00000400:00000010:3.0:1788844903.051900:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001c0ab122. 00000400:00000010:3.0:1788844903.051904:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000506fd682. 00000400:00000010:3.0:1788844903.051906:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000b6b7f8e5. 00000400:00000010:3.0:1788844903.051908:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000019048ef3. 00000400:00000010:3.0:1788844903.051912:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000cce5a98d. 00000400:00000010:3.0:1788844903.051914:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000003242f3c2. 00000400:00000010:3.0:1788844903.051916:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000003f304197. 00000400:00000010:3.0:1788844903.051918:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000016de2d09. 00000400:00000010:3.0:1788844903.051931:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000078e45bfe. 00000400:00000010:3.0:1788844903.051935:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a4bf5b23. 00000400:00000010:3.0:1788844903.051936:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000085a8048d. 00000400:00000010:3.0:1788844903.051938:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000eb39fade. 00000400:00000010:3.0:1788844903.051944:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000073a778bd. 00000400:00000010:3.0:1788844903.051945:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000084c131b0. 00000400:00000010:3.0:1788844903.051947:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000006a949bd. 00000400:00000010:3.0:1788844903.051948:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000dffe2c6. 00000400:00000010:3.0:1788844903.051953:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008608d7f3. 00000400:00000010:3.0:1788844903.051956:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000027ef9691. 00000400:00000010:3.0:1788844903.051962:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005904f260. 00000400:00000010:3.0:1788844903.051964:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003262d3ee. 00000400:00000010:3.0:1788844903.051969:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001c3d3225. 00000400:00000010:3.0:1788844903.051972:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000992ba38a. 00000400:00000010:3.0:1788844903.051973:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000000a5e12c7. 00000400:00000010:3.0:1788844903.051975:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002c4fdf2f. 00000400:00000010:3.0:1788844903.051983:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000048876783. 00000400:00000010:3.0:1788844903.051987:0:69586:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000072b9eea5. 00000400:00000010:3.0:1788844903.051989:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000004ce45170. 00000400:00000010:3.0:1788844903.051991:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001276e185. 00000400:00000010:3.0:1788844903.051994:0:69586:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000beeb68e0. 00000400:00000010:3.0:1788844903.051997:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000007d40e20f. 00000400:00000010:3.0:1788844903.051999:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009a7ce84e. 00000400:00000010:3.0:1788844903.052002:0:69586:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 000000005d80f1e2. 00000400:00000010:3.0:1788844903.052006:0:69586:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 560 at 000000004c36459b. 00000400:00000010:3.0:1788844903.052009:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000024757873. 00000400:00000010:3.0:1788844903.052011:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ab0608cd. 00000400:00000010:3.0:1788844903.052012:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000058f22f4a. 00000400:00000010:3.0:1788844903.052013:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000001188061. 00000400:00000010:3.0:1788844903.052027:0:69586:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 41 at 0000000037c36f6e. 00000400:00000010:3.0:1788844903.052032:0:69586:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000cac47a4a. 00000400:00000010:3.0:1788844903.052045:0:69586:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d7f05c84. 00000400:00000200:3.0:1788844903.052055:0:69586:0:(lib-ptl.c:918:LNetSetLazyPortal()) Setting portal 0 lazy 00000400:00000010:3.0:1788844903.052057:0:69586:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000009eeb148. 00000400:00000010:3.0:1788844903.052061:0:69586:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f667586a. 00000400:00000010:3.0:1788844903.052063:0:69586:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000dc6be67c. 00000400:00000200:3.0:1788844903.052065:0:69586:0:(api-ni.c:2383:lnet_push_target_post()) posted push target 0000000009eeb148 00000400:00000200:3.0:1788844903.052067:0:69586:0:(api-ni.c:2340:lnet_push_target_resize()) nbytes 272 success 00000400:00000010:3.0:1788844903.052070:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000003cc5a3ad. 00000400:00000010:3.0:1788844903.052071:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e31e2148. 00000400:00000010:3.0:1788844903.052073:0:69586:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000004f5dfc7e. 00000400:00000010:3.0:1788844903.052075:0:69586:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d89c206e. 00000400:00000200:2.0:1788844903.056983:0:69586:0:(peer.c:4457:lnet_peer_discovery_start()) discovery start: 0 00000400:00000200:0.0:1788844903.057159:0:69588:0:(peer.c:4283:lnet_peer_discovery()) started 00000400:00000010:1.0:1788844903.073066:0:69589:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 0000000035d83818. 00000400:00000010:1.0:1788844903.073093:0:69589:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 0000000035d83818. 00000400:00000010:1.0:1788844903.073384:0:69589:0:(api-ni.c:7409:lnet_route_show_start()) alloc '(rlist)': 16 at 0000000011597222. 00000400:00000010:1.0:1788844903.073400:0:69589:0:(api-ni.c:7309:lnet_route_show_done()) kfreed 'rlist': 16 at 0000000011597222. 00000400:00000010:1.0:1788844903.073497:0:69589:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 236 at 0000000085b28da0. 00000400:00000010:1.0:1788844903.073505:0:69589:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 236 at 0000000085b28da0. 00000400:00000010:1.0:1788844903.073674:0:69589:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 0000000011597222. 00000400:00000010:1.0:1788844903.073688:0:69589:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 0000000011597222. 00000400:00000010:1.0:1788844903.073763:0:69589:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 12 at 0000000011597222. 00000400:00000010:1.0:1788844903.073768:0:69589:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 12 at 0000000011597222. 00000400:00000010:1.0:1788844903.074330:0:69589:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 32 at 00000000f52232fe. 00000001:00000200:1.0:1788844903.074335:0:69589:0:(udsp.c:863:lnet_udsp_get_policy()) Get UDSP at idx = 0 00000001:00000200:1.0:1788844903.074337:0:69589:0:(udsp.c:878:lnet_udsp_get_policy()) Found UDSP (000000007d9b531d) 00000400:00000010:1.0:1788844903.074339:0:69589:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 32 at 00000000f52232fe. 00000001:02000400:1.0:1788844903.103420:0:69590:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp --if ens2 00000400:00000010:0.0:1788844903.144159:0:69591:0:(api-ni.c:6877:lnet_net_cmd()) alloc '(conf)': 808 at 000000001f4aa745. 00000400:00000010:0.0:1788844903.144178:0:69591:0:(api-ni.c:6555:lnet_genl_parse_local_ni()) alloc '(tun)': 172 at 00000000a393064f. 00000400:00000010:0.0:1788844903.144188:0:69591:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 00000000255b2d0f. 00000400:00000010:0.0:1788844903.144193:0:69591:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 560 at 00000000c7a6e37a. 00000400:00000010:0.0:1788844903.144197:0:69591:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000007078147c. 00000400:00000010:0.0:1788844903.144201:0:69591:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a1603e8c. 00000400:00000010:0.0:1788844903.144204:0:69591:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000003feecb1d. 00000400:00000010:0.0:1788844903.144206:0:69591:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000266a8074. 00000400:00000010:0.0:1788844903.144210:0:69591:0:(config.c:397:lnet_ni_add_interface()) alloc '(ni->ni_interface)': 5 at 000000007d7f717e. 00000400:00000010:0.0:1788844903.144219:0:69591:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000051e05e20. 00000400:00000010:0.0:1788844903.144221:0:69591:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 0000000029a4ebe6. 00000800:00000200:3.0:1788844903.147281:0:69591:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=lo ifindex 1 state 0 ns 000000004dc0f701 00000800:00000200:3.0:1788844903.147291:0:69591:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=lo ifindex 1 state 0 ns 000000004dc0f701 00000800:00000200:3.0:1788844903.147301:0:69591:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=ens2 ifindex 2 state 6 ns 000000004dc0f701 00000800:00000200:3.0:1788844903.147302:0:69591:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=ens2 ifindex 2 state 6 ns 000000004dc0f701 00000800:00000010:3.0:1788844903.147309:0:69591:0:(socklnd.c:2621:ksocknal_startup()) alloc '(net)': 208 at 000000005620bd8e. 00000800:00000200:3.0:1788844903.147322:0:69591:0:(socklnd_modparams.c:295:ksocklnd_lookup_conns_per_peer()) intf ens2 speed -1 00000400:00000200:2.0:1788844903.151310:0:69591:0:(config.c:1657:lnet_get_link_status_locked()) device idx 1 get_link 2 00000400:00000200:2.0:1788844903.151386:0:69591:0:(acceptor.c:209:lnet_acceptor_add_socket()) Added socket on ens2:988 (IP: 192.168.202.29) 00000400:02000000:2.0:1788844903.151402:0:69591:0:(api-ni.c:2823:lnet_startup_lndni()) Added LNI 192.168.202.29@tcp [8/256/0/180] 00000400:00000010:3.0:1788844903.156457:0:69591:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 57 at 0000000043142ca4. 00000400:00000010:3.0:1788844903.156473:0:69591:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000cb47952f. 00000400:00000010:3.0:1788844903.156479:0:69591:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f0840cd. 00000400:00000010:0.0:1788844903.160444:0:69591:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000cac47a4a. 00000400:00000200:0.0:1788844903.160457:0:69591:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d7f05c84 00000400:00000010:0.0:1788844903.160459:0:69591:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000d7f05c84. 00000400:00000010:0.0:1788844903.160465:0:69591:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 41 at 0000000037c36f6e. 00000400:00000010:0.0:1788844903.160542:0:69591:0:(api-ni.c:6852:lnet_genl_parse_local_ni()) kfreed 'tun': 172 at 00000000a393064f. 00000400:00000010:0.0:1788844903.160548:0:69591:0:(api-ni.c:6994:lnet_net_cmd()) kfreed 'conf': 808 at 000000001f4aa745. 00000400:00000010:1.0:1788844903.252034:0:69605:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 0000000035d83818. 00000400:00000010:1.0:1788844903.252097:0:69605:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 0000000035d83818. 00000001:02000400:3.0:1788844904.355620:0:69711:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp1 --if ens2 00000400:00000010:1.0:1788844904.415396:0:69712:0:(api-ni.c:6877:lnet_net_cmd()) alloc '(conf)': 808 at 00000000199c2a60. 00000400:00000010:1.0:1788844904.415417:0:69712:0:(api-ni.c:6555:lnet_genl_parse_local_ni()) alloc '(tun)': 172 at 00000000cd6f6808. 00000400:00000010:1.0:1788844904.415428:0:69712:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 000000000f9bd944. 00000400:00000010:1.0:1788844904.415432:0:69712:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 560 at 000000002d7762e0. 00000400:00000010:1.0:1788844904.415444:0:69712:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000022b887fa. 00000400:00000010:1.0:1788844904.415449:0:69712:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000065964dfc. 00000400:00000010:1.0:1788844904.415451:0:69712:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000000c293bbb. 00000400:00000010:1.0:1788844904.415453:0:69712:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004a4f5b5d. 00000400:00000010:1.0:1788844904.415457:0:69712:0:(config.c:397:lnet_ni_add_interface()) alloc '(ni->ni_interface)': 5 at 0000000035d83818. 00000800:00000010:1.0:1788844904.415466:0:69712:0:(socklnd.c:2621:ksocknal_startup()) alloc '(net)': 208 at 0000000085b28da0. 00000800:00000200:1.0:1788844904.415478:0:69712:0:(socklnd_modparams.c:295:ksocklnd_lookup_conns_per_peer()) intf ens2 speed -1 00000400:00000200:1.0:1788844904.415485:0:69712:0:(config.c:1657:lnet_get_link_status_locked()) device idx 1 get_link 2 00000400:00000200:1.0:1788844904.416664:0:69712:0:(acceptor.c:209:lnet_acceptor_add_socket()) Added socket on ens2:988 (IP: 192.168.202.29) 00000400:02000000:1.0:1788844904.416703:0:69712:0:(api-ni.c:2823:lnet_startup_lndni()) Added LNI 192.168.202.29@tcp1 [8/256/0/180] 00000400:00000010:1.0:1788844904.430459:0:69712:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 73 at 000000002a657409. 00000400:00000010:1.0:1788844904.430473:0:69712:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000007d2552db. 00000400:00000010:1.0:1788844904.430507:0:69712:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e8f88037. 00000400:00000010:1.0:1788844904.430533:0:69712:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000cb47952f. 00000400:00000200:1.0:1788844904.430536:0:69712:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f0840cd 00000400:00000010:1.0:1788844904.430538:0:69712:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000007f0840cd. 00000400:00000010:1.0:1788844904.430543:0:69712:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 57 at 0000000043142ca4. 00000400:00000010:1.0:1788844904.430558:0:69712:0:(api-ni.c:6852:lnet_genl_parse_local_ni()) kfreed 'tun': 172 at 00000000cd6f6808. 00000400:00000010:1.0:1788844904.430564:0:69712:0:(api-ni.c:6994:lnet_net_cmd()) kfreed 'conf': 808 at 00000000199c2a60. 00000400:00000010:3.0:1788844904.581678:0:69717:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000001a29c2bd. 00000400:00000010:3.0:1788844904.581747:0:69717:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000001a29c2bd. 00000400:00000010:1.0:1788844905.872762:0:69836:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000ef4282cd. 00000400:00000010:1.0:1788844905.872887:0:69836:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000ef4282cd. 00000001:02000400:3.0:1788844905.893547:0:69838:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl discover 192.168.202.29@tcp 00000400:00000010:0.0:1788844905.912025:0:69839:0:(peer.c:230:lnet_peer_alloc()) alloc '(lp)': 360 at 00000000c22a5afc. 00000400:00000200:0.0:1788844905.912041:0:69839:0:(peer.c:266:lnet_peer_alloc()) 00000000c22a5afc nid 192.168.202.29@tcp 00000400:00000010:0.0:1788844905.912045:0:69839:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 0000000080339687. 00000400:00000200:0.0:1788844905.912049:0:69839:0:(peer.c:203:lnet_peer_net_alloc()) 0000000080339687 net tcp 00000400:00000010:0.0:1788844905.912051:0:69839:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 0000000061c20f74. 00000400:00000200:0.0:1788844905.912054:0:69839:0:(peer.c:184:lnet_peer_ni_alloc()) 0000000061c20f74 nid 192.168.202.29@tcp 00000400:00000200:0.0:1788844905.912060:0:69839:0:(peer.c:1804:lnet_peer_attach_peer_ni()) peer 192.168.202.29@tcp NID 192.168.202.29@tcp flags 0x0 00000400:00000200:0.0:1788844905.912071:0:69839:0:(peer.c:2183:lnet_peer_ni_traffic_add()) peer 192.168.202.29@tcp: 0 00000400:00000200:0.0:1788844905.912082:0:69839:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.202.29@tcp: 0 00000400:00000200:0.0:1788844905.912085:0:69839:0:(peer.c:2935:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1788844905.912186:0:69588:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1788844905.912191:0:69588:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.202.29@tcp(00000000c22a5afc) state 0x40 00000400:00000010:1.0:1788844905.912195:0:69588:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000003f62c563. 00000400:00000010:1.0:1788844905.912199:0:69588:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001fe8b3ec. 00000400:00000010:1.0:1788844905.912218:0:69588:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000d49c0485. 00000400:00000200:1.0:1788844905.912222:0:69588:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.29@tcp 00000400:00000200:1.0:1788844905.912232:0:69588:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.29@tcp local destination 00000400:00000200:1.0:1788844905.912237:0:69588:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.29@tcp 00000400:00000200:1.0:1788844905.912239:0:69588:0:(lib-move.c:1874:lnet_set_non_mr_pref_nid()) Setting preferred local NID 192.168.202.29@tcp on NMR peer 192.168.202.29@tcp 00000400:00000200:1.0:1788844905.912242:0:69588:0:(peer.c:1171:lnet_peer_ni_set_non_mr_pref_nid()) peer 192.168.202.29@tcp nid 192.168.202.29@tcp: 0 00000400:00000200:1.0:1788844905.912245:0:69588:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [18:18:256:4294967295] 192.168.202.29@tcp LPNI seq info [1:1:8:4294967295] 00000400:00000200:1.0:1788844905.912250:0:69588:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:1.0:1788844905.912258:0:69588:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : GET try# 0 00000800:00000200:1.0:1788844905.912262:0:69588:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.29@tcp 00000800:00000010:1.0:1788844905.912266:0:69588:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000007dd3539c. 00000800:00000010:1.0:1788844905.912272:0:69588:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 00000000cd6f6808. 00000800:00000010:1.0:1788844905.912275:0:69588:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 00000000e735332f. 00000800:00000200:1.0:1788844905.912278:0:69588:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000cd6f6808] -> 12345-192.168.202.29@tcp (2) 00000800:00000200:1.1:1788844905.912281:0:69588:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000cd6f6808] -> 12345-192.168.202.29@tcp (2) 00000400:00000200:1.0:1788844905.912297:0:69588:0:(peer.c:4022:lnet_peer_send_ping()) peer 192.168.202.29@tcp 00000400:00000200:1.0:1788844905.912299:0:69588:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.202.29@tcp(00000000c22a5afc) state 0x240 rc 0 00000800:00000010:2.0:1788844905.912600:0:69593:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 000000006ade1324. 00000800:00000010:2.0:1788844905.912605:0:69593:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 0000000012c70bc9. 00000800:00000010:2.0:1788844905.912609:0:69593:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000aa794c31. 00000800:00000010:2.0:1788844905.912628:0:69593:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000aa794c31. 00000400:00000200:1.0:1788844905.912825:0:69600:0:(acceptor.c:578:lnet_acceptor()) Accepted connection on ens2:988 00000400:00000200:1.0:1788844905.912840:0:69600:0:(acceptor.c:502:lnet_accept()) Accept 192.168.202.29@tcp from 255.3.0.2 00000800:00000010:1.0:1788844905.912844:0:69600:0:(socklnd.c:737:ksocknal_accept()) alloc '(cr)': 32 at 0000000023d9ee36. 00000800:00000010:0.0:1788844905.913004:0:69592:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 000000008feb9acd. 00000800:00000010:0.0:1788844905.913008:0:69592:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 000000006e5c1771. 00000800:00000010:0.0:1788844905.913022:0:69592:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 000000006fc81c16. 00000800:00000200:0.1:1788844905.913027:0:69592:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000cd6f6808] -> 12345-192.168.202.29@tcp (2) 00000800:00000200:0.1:1788844905.913031:0:69592:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.202.29@tcp 000000006fc81c16 deleted 00000800:00000010:0.1:1788844905.913034:0:69592:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 000000006fc81c16. 00000800:00000200:0.1:1788844905.913039:0:69592:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:0.1:1788844905.913045:0:69592:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000200:0.1:1788844905.913050:0:69592:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000007dd3539c type 2, nob 96 niov 1 nkiov 0 00000800:00000200:0.0:1788844905.913055:0:69592:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.202.29@tcp p 3.x 192.168.202.29 -> 192.168.202.29:1023 incarnation:1788844903147311150 sched[0] 00000800:00000010:0.0:1788844905.913060:0:69592:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 000000003911e19e. 00000800:00000010:0.0:1788844905.913119:0:69592:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 000000003911e19e. 00000800:00000010:0.0:1788844905.913123:0:69592:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 000000006e5c1771. 00000800:00000010:0.0:1788844905.913152:0:69592:0:(socklnd_cb.c:2179:ksocknal_connd()) kfreed 'cr': 32 at 0000000023d9ee36. 00000800:00000200:0.0:1788844905.913203:0:69598:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788844905.913207:0:69598:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000007dd3539c. 00000400:00000200:0.0:1788844905.913211:0:69598:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788844905.913216:0:69598:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: GET: OK 00000400:00000200:0.0:1788844905.913221:0:69598:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1788844905.913224:0:69598:0:(peer.c:3241:lnet_discovery_event_send()) Ping Send to 192.168.202.29@tcp: 0 00000800:00000200:2.1:1788844905.913292:0:69593:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:2.0:1788844905.913298:0:69593:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.202.29@tcp p 3.x 192.168.202.29 -> 192.168.202.29:988 incarnation:1788844903147311150 sched[0] 00000800:00000010:2.0:1788844905.913301:0:69593:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 0000000012c70bc9. 00000400:00000200:2.0:1788844905.913352:0:69593:0:(lib-socket.c:438:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.202.29:988 00000800:00000010:2.0:1788844905.913495:0:69593:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 00000000a5ec57bb. 00000800:00000010:2.0:1788844905.913512:0:69593:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 0000000012c70bc9. 00000800:00000010:2.0:1788844905.913516:0:69593:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000aa794c31. 00000800:00000010:2.0:1788844905.913544:0:69593:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000aa794c31. 00000400:00000200:2.0:1788844905.913569:0:69600:0:(acceptor.c:578:lnet_acceptor()) Accepted connection on ens2:988 00000400:00000200:2.0:1788844905.913578:0:69600:0:(acceptor.c:502:lnet_accept()) Accept 192.168.202.29@tcp from 254.3.0.2 00000800:00000010:2.0:1788844905.913580:0:69600:0:(socklnd.c:737:ksocknal_accept()) alloc '(cr)': 32 at 000000000c01a2c2. 00000400:00000200:0.0:1788844905.913648:0:69599:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : GET - for me 00000400:00000200:0.0:1788844905.913673:0:69599:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.29@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1788844905.913681:0:69599:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.29@tcp of length 64/272 into md 0xd [1] + 0 00000400:00000200:0.0:1788844905.913693:0:69599:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.29@tcp to NMR: 192.168.202.29@tcp local destination 00000400:00000200:0.0:1788844905.913697:0:69599:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [19:19:256:4294967295] 192.168.202.29@tcp LPNI seq info [2:2:8:4294967295] 00000400:00000200:0.0:1788844905.913705:0:69599:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) 192.168.202.29@tcp : REPLY try# 0 00000800:00000200:0.0:1788844905.913710:0:69599:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.202.29@tcp 00000800:00000010:0.0:1788844905.913716:0:69599:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000007f394097. 00000800:00000200:0.0:1788844905.913720:0:69599:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000cd6f6808] -> 12345-192.168.202.29@tcp (4) 00000800:00000200:0.0:1788844905.913724:0:69599:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000200:0.0:1788844905.913727:0:69599:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000007f394097 type 3, nob 160 niov 1 nkiov 1 00000800:00000200:0.0:1788844905.913776:0:69599:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788844905.913811:0:69599:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000007f394097. 00000400:00000200:0.0:1788844905.913814:0:69599:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788844905.913818:0:69599:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: REPLY: OK 00000400:00000200:0.0:1788844905.913835:0:69599:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : REPLY - for me 00000400:00000200:0.0:1788844905.913843:0:69599:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.29@tcp: Reply from 12345-192.168.202.29@tcp of length 64/64 into md 0x11 00000400:00000200:0.0:1788844905.913849:0:69599:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788844905.913851:0:69599:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: REPLY: OK 00000400:00000200:0.0:1788844905.913855:0:69599:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001fe8b3ec 00000400:00000010:0.0:1788844905.913857:0:69599:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000001fe8b3ec. 00000400:00000200:0.0:1788844905.913861:0:69599:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 3 00000400:00000200:0.0:1788844905.913864:0:69599:0:(peer.c:3119:lnet_discovery_event_reply()) Peer 192.168.202.29@tcp has discovery enabled 00000400:00000200:0.0:1788844905.913867:0:69599:0:(peer.c:3145:lnet_discovery_event_reply()) peer 192.168.202.29@tcp(00000000c22a5afc) is MR capable 00000400:00000200:0.0:1788844905.913870:0:69599:0:(peer.c:1196:lnet_peer_ni_clr_non_mr_pref_nid()) peer 192.168.202.29@tcp: 0 00000400:00000200:0.0:1788844905.913874:0:69599:0:(peer.c:3200:lnet_discovery_event_reply()) peer 192.168.202.29@tcp data present 9. state = 0x241 00000400:00000200:0.0:1788844905.913877:0:69599:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.202.29@tcp: -114 00000400:00000200:1.0:1788844905.914021:0:69588:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1788844905.914027:0:69588:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.202.29@tcp(00000000c22a5afc) state 0xc1 00000400:00000010:1.0:1788844905.914034:0:69588:0:(peer.c:3474:lnet_peer_merge_data()) alloc '(curnis)': 60 at 00000000dcc2f31c. 00000400:00000010:1.0:1788844905.914037:0:69588:0:(peer.c:3475:lnet_peer_merge_data()) alloc '(addnis)': 72 at 000000001a0eaf9f. 00000400:00000010:1.0:1788844905.914039:0:69588:0:(peer.c:3476:lnet_peer_merge_data()) alloc '(delnis)': 60 at 00000000b5b154a4. 00000400:00000010:1.0:1788844905.914046:0:69588:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 000000000d8bd57e. 00000400:00000200:1.0:1788844905.914050:0:69588:0:(peer.c:184:lnet_peer_ni_alloc()) 000000000d8bd57e nid 192.168.202.29@tcp1 00000400:00000010:1.0:1788844905.914052:0:69588:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 00000000856394e9. 00000400:00000200:1.0:1788844905.914054:0:69588:0:(peer.c:203:lnet_peer_net_alloc()) 00000000856394e9 net tcp1 00000400:00000200:1.0:1788844905.914060:0:69588:0:(peer.c:1804:lnet_peer_attach_peer_ni()) peer 192.168.202.29@tcp NID 192.168.202.29@tcp1 flags 0x11 00000400:00000010:1.0:1788844905.914063:0:69588:0:(peer.c:3623:lnet_peer_merge_data()) kfreed 'curnis': 60 at 00000000dcc2f31c. 00000400:00000010:1.0:1788844905.914066:0:69588:0:(peer.c:3624:lnet_peer_merge_data()) kfreed 'addnis': 72 at 000000001a0eaf9f. 00000400:00000010:1.0:1788844905.914068:0:69588:0:(peer.c:3625:lnet_peer_merge_data()) kfreed 'delnis': 60 at 00000000b5b154a4. 00000400:00000010:1.0:1788844905.914071:0:69588:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000003f62c563. 00000400:00000200:1.0:1788844905.914074:0:69588:0:(peer.c:3628:lnet_peer_merge_data()) peer 192.168.202.29@tcp (00000000c22a5afc): 0 00000400:00000200:1.0:1788844905.914077:0:69588:0:(peer.c:3940:lnet_peer_data_present()) peer 192.168.202.29@tcp(00000000c22a5afc): 0. state = 0x141 00000400:00000200:1.0:1788844905.914080:0:69588:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.202.29@tcp(00000000c22a5afc) state 0x141 rc 1 00000400:00000200:1.0:1788844905.914083:0:69588:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.202.29@tcp(00000000c22a5afc) state 0x141 00000400:00000010:1.0:1788844905.914088:0:69588:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000001ef93f4. 00000400:00000010:1.0:1788844905.914095:0:69588:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000007bd1e39d. 00000400:00000200:1.0:1788844905.914099:0:69588:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.29@tcp 00000400:00000200:1.0:1788844905.914105:0:69588:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.29@tcp to MR: 192.168.202.29@tcp local destination 00000400:00000200:1.0:1788844905.914115:0:69588:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [20:20:256:4294967295] 192.168.202.29@tcp LPNI seq info [3:3:8:4294967295] 00000400:00000200:1.0:1788844905.914119:0:69588:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:1.0:1788844905.914124:0:69588:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : PUT try# 0 00000800:00000200:1.0:1788844905.914129:0:69588:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.202.29@tcp 00000800:00000010:1.0:1788844905.914132:0:69588:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000b810af09. 00000800:00000200:1.0:1788844905.914136:0:69588:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000cd6f6808] -> 12345-192.168.202.29@tcp (4) 00000800:00000200:1.0:1788844905.914139:0:69588:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000200:1.0:1788844905.914142:0:69588:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000b810af09 type 1, nob 160 niov 1 nkiov 1 00000400:00000200:1.0:1788844905.914152:0:69588:0:(peer.c:4164:lnet_peer_send_push()) peer 192.168.202.29@tcp 00000400:00000200:1.0:1788844905.914154:0:69588:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.202.29@tcp(00000000c22a5afc) state 0x541 rc 0 00000800:00000200:1.0:1788844905.914199:0:69599:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1788844905.914202:0:69599:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000b810af09. 00000400:00000200:1.0:1788844905.914205:0:69599:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1788844905.914208:0:69599:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: PUT: OK 00000400:00000200:1.0:1788844905.914210:0:69599:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1788844905.914212:0:69599:0:(peer.c:3241:lnet_discovery_event_send()) Push Send to 192.168.202.29@tcp: 0 00000400:00000200:1.0:1788844905.914222:0:69599:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : PUT - for me 00000400:00000200:1.0:1788844905.914229:0:69599:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.29@tcp of length 64 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1788844905.914233:0:69599:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 0 from 12345-192.168.202.29@tcp of length 64/64 into md 0x5 [1] + 0 00000400:00000010:1.0:1788844905.914237:0:69599:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f667586a. 00000400:00000200:1.0:1788844905.914240:0:69599:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000dc6be67c 00000400:00000200:1.0:1788844905.914246:0:69599:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1788844905.914249:0:69599:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: PUT: OK 00000400:00000200:1.0:1788844905.914251:0:69599:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000dc6be67c 00000400:00000010:1.0:1788844905.914253:0:69599:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000dc6be67c. 00000400:00000200:1.0:1788844905.914255:0:69599:0:(api-ni.c:2393:lnet_push_target_event_handler()) type 2 status 0 unlinked 1 00000400:00000200:1.0:1788844905.914260:0:69599:0:(peer.c:2778:lnet_peer_push_event()) peer 192.168.202.29@tcp(00000000c22a5afc) is MR 00000400:00000010:1.0:1788844905.914263:0:69599:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 73 at 000000001a0eaf9f. 00000400:00000200:1.0:1788844905.914266:0:69599:0:(peer.c:2852:lnet_peer_push_event()) Received Push 192.168.202.29@tcp 9 00000400:00000200:1.0:1788844905.914269:0:69599:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.202.29@tcp: -114 00000400:00000200:1.0:1788844905.914283:0:69599:0:(lib-md.c:96:lnet_cpt_of_md()) Couldn't resolve first page of md 0000000000000000 with offset 0 00000400:00000200:1.0:1788844905.914288:0:69599:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.29@tcp to MR: 192.168.202.29@tcp local destination 00000400:00000200:1.0:1788844905.914291:0:69599:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [21:21:256:4294967295] 192.168.202.29@tcp LPNI seq info [4:4:8:4294967295] 00000400:00000200:1.0:1788844905.914298:0:69599:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) 192.168.202.29@tcp : ACK try# 0 00000800:00000200:1.0:1788844905.914302:0:69599:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 1 frags to 12345-192.168.202.29@tcp 00000800:00000010:1.0:1788844905.914305:0:69599:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000b810af09. 00000800:00000200:1.0:1788844905.914309:0:69599:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000cd6f6808] -> 12345-192.168.202.29@tcp (4) 00000800:00000200:1.0:1788844905.914313:0:69599:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000200:1.0:1788844905.914315:0:69599:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000b810af09 type 0, nob 96 niov 1 nkiov 0 00000800:00000200:1.0:1788844905.914345:0:69599:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1788844905.914347:0:69599:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000b810af09. 00000400:00000200:1.0:1788844905.914350:0:69599:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1788844905.914353:0:69599:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: ACK: OK 00000400:00000200:1.0:1788844905.914364:0:69599:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : ACK - for me 00000400:00000200:1.0:1788844905.914371:0:69599:0:(lib-move.c:4539:lnet_parse_ack()) 192.168.202.29@tcp: ACK from 12345-192.168.202.29@tcp into md 0x15 00000400:00000200:1.0:1788844905.914374:0:69599:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1788844905.914377:0:69599:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: ACK: OK 00000400:00000200:1.0:1788844905.914379:0:69599:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000001ef93f4 00000400:00000010:1.0:1788844905.914381:0:69599:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 0000000001ef93f4. 00000400:00000200:1.0:1788844905.914383:0:69599:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 4 00000400:00000200:1.0:1788844905.914386:0:69599:0:(peer.c:3007:lnet_discovery_event_ack()) peer 192.168.202.29@tcp ev->status 0 00000400:00000200:1.0:1788844905.914388:0:69599:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.202.29@tcp: -114 00000400:00000200:1.0:1788844905.914400:0:69588:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000010:1.0:1788844905.914404:0:69588:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d032620a. 00000400:00000010:1.0:1788844905.914407:0:69588:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000001ef93f4. 00000400:00000200:1.0:1788844905.914412:0:69588:0:(api-ni.c:2383:lnet_push_target_post()) posted push target 0000000009eeb148 00000400:00000200:1.0:1788844905.914414:0:69588:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.202.29@tcp(00000000c22a5afc) state 0xc1 00000400:00000010:1.0:1788844905.914418:0:69588:0:(peer.c:3474:lnet_peer_merge_data()) alloc '(curnis)': 60 at 00000000b5b154a4. 00000400:00000010:1.0:1788844905.914420:0:69588:0:(peer.c:3475:lnet_peer_merge_data()) alloc '(addnis)': 72 at 00000000d44e5eb4. 00000400:00000010:1.0:1788844905.914422:0:69588:0:(peer.c:3476:lnet_peer_merge_data()) alloc '(delnis)': 60 at 00000000dcc2f31c. 00000400:00000010:1.0:1788844905.914425:0:69588:0:(peer.c:3623:lnet_peer_merge_data()) kfreed 'curnis': 60 at 00000000b5b154a4. 00000400:00000010:1.0:1788844905.914432:0:69588:0:(peer.c:3624:lnet_peer_merge_data()) kfreed 'addnis': 72 at 00000000d44e5eb4. 00000400:00000010:1.0:1788844905.914434:0:69588:0:(peer.c:3625:lnet_peer_merge_data()) kfreed 'delnis': 60 at 00000000dcc2f31c. 00000400:00000010:1.0:1788844905.914436:0:69588:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 73 at 000000001a0eaf9f. 00000400:00000200:1.0:1788844905.914439:0:69588:0:(peer.c:3628:lnet_peer_merge_data()) peer 192.168.202.29@tcp (00000000c22a5afc): 0 00000400:00000200:1.0:1788844905.914441:0:69588:0:(peer.c:3940:lnet_peer_data_present()) peer 192.168.202.29@tcp(00000000c22a5afc): 0. state = 0x141 00000400:00000200:1.0:1788844905.914443:0:69588:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.202.29@tcp(00000000c22a5afc) state 0x141 rc 1 00000400:00000200:1.0:1788844905.914446:0:69588:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.202.29@tcp(00000000c22a5afc) state 0x141 00000400:00000200:1.0:1788844905.914449:0:69588:0:(peer.c:4082:lnet_peer_discovered()) peer 192.168.202.29@tcp 00000400:00000200:1.0:1788844905.914451:0:69588:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.202.29@tcp(00000000c22a5afc) state 0x111 rc 0 00000400:00000200:1.0:1788844905.914454:0:69588:0:(peer.c:2629:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 192.168.202.29@tcp 00000400:00000200:0.0:1788844905.914605:0:69839:0:(peer.c:2984:lnet_discover_peer_locked()) peer 192.168.202.29@tcp NID 192.168.202.29@tcp: 0. discovery complete 00000800:00000010:3.0:1788844905.915617:0:69594:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 0000000071fab5be. 00000800:00000010:3.0:1788844905.915626:0:69594:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 0000000016f26155. 00000800:00000010:3.0:1788844905.915644:0:69594:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 00000000cf4d4ded. 00000800:00000200:3.1:1788844905.915651:0:69594:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000cd6f6808] -> 12345-192.168.202.29@tcp (4) 00000800:00000200:3.1:1788844905.915655:0:69594:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.202.29@tcp 00000000cf4d4ded deleted 00000800:00000010:3.1:1788844905.915657:0:69594:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 00000000cf4d4ded. 00000800:00000200:3.1:1788844905.915661:0:69594:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 3, ksnr_connected a ksnr_max_conns 1 00000800:00000200:3.0:1788844905.915666:0:69594:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.202.29@tcp p 3.x 192.168.202.29 -> 192.168.202.29:1022 incarnation:1788844903147311150 sched[0] 00000800:00000010:3.0:1788844905.915670:0:69594:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000aa726205. 00000800:00000010:3.0:1788844905.915766:0:69594:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000aa726205. 00000800:00000010:3.0:1788844905.915773:0:69594:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 0000000016f26155. 00000800:00000010:3.0:1788844905.915874:0:69594:0:(socklnd_cb.c:2179:ksocknal_connd()) kfreed 'cr': 32 at 000000000c01a2c2. 00000800:00000200:3.1:1788844905.915895:0:69593:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 2, ksnr_connected e ksnr_max_conns 1 00000800:00000200:3.0:1788844905.915901:0:69593:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.202.29@tcp p 3.x 192.168.202.29 -> 192.168.202.29:988 incarnation:1788844903147311150 sched[0] 00000800:00000010:3.0:1788844905.915903:0:69593:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 0000000012c70bc9. 00000400:00000010:0.0:1788844905.928989:0:69841:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 12 at 000000002bfd01db. 00000400:00000010:0.0:1788844905.929004:0:69841:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 12 at 000000002bfd01db. 00000001:02000400:2.0:1788844905.946435:0:69843:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 10 00000001:02000400:2.0:1788844905.984557:0:69845:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp -d *@tcp -m GET -r 1 -e remote_error 00000400:00000200:3.0:1788844906.009413:0:69846:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844906.009423:0:69846:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844906.009434:0:69846:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp 00000400:00000200:3.0:1788844906.009435:0:69846:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844906.009438:0:69846:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp 00000400:00000200:3.0:1788844906.009439:0:69846:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844906.009441:0:69846:0:(api-ni.c:3207:nla_strnid()) -> 00000400:00000200:3.0:1788844906.009442:0:69846:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844906.009443:0:69846:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844906.009444:0:69846:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844906.009446:0:69846:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844906.009447:0:69846:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844906.009456:0:69846:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844906.009458:0:69846:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000010:3.0:1788844906.009461:0:69846:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 0000000086baf2f1. 00000400:00000200:3.0:1788844906.009469:0:69846:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp, dst 255.255.255.255@tcp, rate 1, interval 0 00000001:02000400:3.0:1788844906.021469:0:69847:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net_drop add -s *@tcp1 -d *@tcp1 -m GET -r 1 -e remote_error 00000400:00000200:3.0:1788844906.042078:0:69848:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844906.042088:0:69848:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844906.042102:0:69848:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1 00000400:00000200:3.0:1788844906.042104:0:69848:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844906.042106:0:69848:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1 00000400:00000200:3.0:1788844906.042108:0:69848:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844906.042110:0:69848:0:(api-ni.c:3207:nla_strnid()) -> 00000400:00000200:3.0:1788844906.042111:0:69848:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844906.042112:0:69848:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844906.042114:0:69848:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844906.042116:0:69848:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844906.042117:0:69848:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844906.042119:0:69848:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844906.042121:0:69848:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000010:3.0:1788844906.042125:0:69848:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 00000000141ad680. 00000400:00000200:3.0:1788844906.042133:0:69848:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp1, dst 255.255.255.255@tcp1, rate 1, interval 0 00000001:02000400:1.0:1788844906.055229:0:69849:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp -d *@tcp -r 1 00000400:00000200:0.0:1788844906.083488:0:69850:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844906.083510:0:69850:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844906.083522:0:69850:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp 00000400:00000200:0.0:1788844906.083524:0:69850:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844906.083527:0:69850:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp 00000400:00000200:0.0:1788844906.083528:0:69850:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844906.083530:0:69850:0:(api-ni.c:3207:nla_strnid()) -> 00000400:00000200:0.0:1788844906.083532:0:69850:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844906.083533:0:69850:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844906.083534:0:69850:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844906.083536:0:69850:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844906.083537:0:69850:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844906.083539:0:69850:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844906.083541:0:69850:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000010:0.0:1788844906.083545:0:69850:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 00000000f48013a0. 00000400:00000200:0.0:1788844906.083554:0:69850:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp, dst 255.255.255.255@tcp, rate 1, interval 0 00000001:02000400:2.0:1788844906.100946:0:69851:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net_drop add -s *@tcp1 -d *@tcp1 -r 1 00000400:00000200:0.0:1788844906.122578:0:69852:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844906.122590:0:69852:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844906.122610:0:69852:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1 00000400:00000200:0.0:1788844906.122611:0:69852:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844906.122614:0:69852:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1 00000400:00000200:0.0:1788844906.122615:0:69852:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844906.122617:0:69852:0:(api-ni.c:3207:nla_strnid()) -> 00000400:00000200:0.0:1788844906.122618:0:69852:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844906.122620:0:69852:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844906.122621:0:69852:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844906.122623:0:69852:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844906.122624:0:69852:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844906.122625:0:69852:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844906.122627:0:69852:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000010:0.0:1788844906.122631:0:69852:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 000000009a9f514e. 00000400:00000200:0.0:1788844906.122639:0:69852:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp1, dst 255.255.255.255@tcp1, rate 1, interval 0 00000001:02000400:3.0:1788844906.136580:0:69853:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --nid 192.168.202.29@tcp --health 0 00000400:00000200:0.0:1788844906.165090:0:69854:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.202.29@tcp added to recovery queue. ping count: 0 next ping: 1470 last alive: 1469 health: 0 00000400:00000200:3.0:1788844906.213194:0:69860:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:3.0:1788844906.213203:0:69860:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000aa726205. 00000400:00000200:3.0:1788844906.213213:0:69860:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:3.0:1788844906.213231:0:69860:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 000000001b41631b. 00000400:00000200:3.0:1788844906.213237:0:69860:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:3.0:1788844906.213241:0:69860:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 000000001b41631b. 00000400:00000010:3.0:1788844906.213251:0:69860:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000aa726205. 00000001:02000400:0.0:1788844906.259469:0:69862:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p 00000400:00000200:2.0:1788844906.286072:0:69863:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:2.0:1788844906.286082:0:69863:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000070a783f. 00000400:00000200:2.0:1788844906.286093:0:69863:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:2.0:1788844906.286112:0:69863:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000ebd2e1d2. 00000400:00000200:2.0:1788844906.286118:0:69863:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:2.0:1788844906.286122:0:69863:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000ebd2e1d2. 00000400:00000010:2.0:1788844906.286134:0:69863:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000070a783f. 00000400:00000200:3.0:1788844906.361712:0:69869:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:3.0:1788844906.361722:0:69869:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000aa726205. 00000400:00000200:3.0:1788844906.361734:0:69869:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:3.0:1788844906.362040:0:69869:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 000000001b41631b. 00000400:00000200:3.0:1788844906.362049:0:69869:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:3.0:1788844906.362053:0:69869:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 000000001b41631b. 00000400:00000010:3.0:1788844906.362064:0:69869:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000aa726205. 00000400:00000010:0.0:1788844907.160129:0:69587:0:(lib-move.c:3721:lnet_recover_peer_nis()) alloc '(ev_info)': 24 at 00000000fc09b53f. 00000400:00000010:0.0:1788844907.160146:0:69587:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000714772fe. 00000400:00000010:0.0:1788844907.160175:0:69587:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cbe4bce4. 00000400:00000010:0.0:1788844907.160185:0:69587:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000d0711c20. 00000400:00000200:0.0:1788844907.160193:0:69587:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.29@tcp 00000400:00000200:0.0:1788844907.160206:0:69587:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.29@tcp local destination 00000400:00000200:0.0:1788844907.160212:0:69587:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.29@tcp 00000400:00000200:0.0:1788844907.160217:0:69587:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [22:22:256:4294967295] 192.168.202.29@tcp LPNI seq info [5:5:8:4294967295] 00000400:00000200:0.0:1788844907.160224:0:69587:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:0.0:1788844907.160231:0:69587:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : GET try# 0 00000800:00000200:0.0:1788844907.160237:0:69587:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.29@tcp 00000800:00000010:0.0:1788844907.160242:0:69587:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000007f394097. 00000800:00000200:0.0:1788844907.160248:0:69587:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000cd6f6808] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:0.0:1788844907.160257:0:69587:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000200:0.0:1788844907.160261:0:69587:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000007f394097 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:0.0:1788844907.160286:0:69587:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.202.29@tcp added to recovery queue. ping count: 1 next ping: 1472 last alive: 1469 health: 0 00000400:00000010:0.0:1788844907.160292:0:69587:0:(lib-move.c:3814:lnet_print_recovery_list()) alloc '(buf)': 65 at 0000000094a5592f. 00000400:02000040:0.0:1788844907.160302:0:69587:0:(lib-move.c:3830:lnet_print_recovery_list()) 1 peer NIs in recovery (showing 1): 192.168.202.29@tcp 00000400:00000200:2.0:1788844907.160776:0:69598:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 1 00000400:00000200:2.0:1788844907.160829:0:69598:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.202.29@tcp(192.168.202.29@tcp)->dst 192.168.202.29@tcp: GET simulate health error: REMOTE_ERROR 00000800:02000400:2.0:1788844907.160833:0:69598:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.202.29: rc = -22 00000400:00000010:0.0:1788844907.174852:0:69587:0:(lib-move.c:3832:lnet_print_recovery_list()) kfreed 'buf': 65 at 0000000094a5592f. 00000800:00000200:2.0:1788844907.195380:0:69598:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [000000006ade1324] Error -22 on write to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000010:2.0:1788844907.195390:0:69598:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000007f394097. 00000400:00000200:2.0:1788844907.195398:0:69598:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788844907.195403:0:69598:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1788844907.195412:0:69598:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5 00000400:00000200:2.0:1788844907.195416:0:69598:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.29@tcp recovery message sent unsuccessfully:-5 00000400:00000200:2.0:1788844907.195421:0:69598:0:(lib-move.c:4104:lnet_handle_recovery_reply()) peer NI (192.168.202.29@tcp) recovery failed with -5 00000400:00000200:1.0:1788844907.477378:0:69877:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:1.0:1788844907.477389:0:69877:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000d62b204e. 00000400:00000200:1.0:1788844907.477400:0:69877:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:1.0:1788844907.477419:0:69877:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 000000009e60f96b. 00000400:00000200:1.0:1788844907.477427:0:69877:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:1.0:1788844907.477431:0:69877:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 000000009e60f96b. 00000400:00000010:1.0:1788844907.477439:0:69877:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000d62b204e. 00000400:00000200:0.0:1788844908.184277:0:69587:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cbe4bce4 00000400:00000010:0.0:1788844908.184298:0:69587:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000cbe4bce4. 00000400:00000200:0.0:1788844908.184306:0:69587:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0 00000400:00000200:0.0:1788844908.184313:0:69587:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.202.29@tcp recovery ping unlinked 00000400:00000010:0.0:1788844908.184318:0:69587:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000fc09b53f. 00000400:00000010:0.0:1788844908.184323:0:69587:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000714772fe. 00000400:00000200:3.0:1788844908.637022:0:69885:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:3.0:1788844908.637032:0:69885:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000001b41631b. 00000400:00000200:3.0:1788844908.637043:0:69885:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:3.0:1788844908.637062:0:69885:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000a0cabecd. 00000400:00000200:3.0:1788844908.637069:0:69885:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:3.0:1788844908.637073:0:69885:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000a0cabecd. 00000400:00000010:3.0:1788844908.637083:0:69885:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000001b41631b. 00000400:00000010:0.0:1788844909.208144:0:69587:0:(lib-move.c:3721:lnet_recover_peer_nis()) alloc '(ev_info)': 24 at 000000003f067e94. 00000400:00000010:0.0:1788844909.208160:0:69587:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000622eb0be. 00000400:00000010:0.0:1788844909.208166:0:69587:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cbe4bce4. 00000400:00000010:0.0:1788844909.208175:0:69587:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000034fcdbe8. 00000400:00000200:0.0:1788844909.208183:0:69587:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.29@tcp 00000400:00000200:0.0:1788844909.208197:0:69587:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.29@tcp local destination 00000400:00000200:0.0:1788844909.208204:0:69587:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.29@tcp 00000400:00000200:0.0:1788844909.208208:0:69587:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [23:23:256:4294967295] 192.168.202.29@tcp LPNI seq info [6:6:8:4294967295] 00000400:00000200:0.0:1788844909.208215:0:69587:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:0.0:1788844909.208222:0:69587:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : GET try# 0 00000800:00000200:0.0:1788844909.208228:0:69587:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.29@tcp 00000800:00000010:0.0:1788844909.208233:0:69587:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000046d74fc0. 00000800:00000200:0.0:1788844909.208238:0:69587:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000cd6f6808] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:0.0:1788844909.208246:0:69587:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000200:0.0:1788844909.208250:0:69587:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000046d74fc0 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:0.0:1788844909.208272:0:69587:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.202.29@tcp added to recovery queue. ping count: 2 next ping: 1476 last alive: 1469 health: 0 00000400:00000200:1.0:1788844909.213181:0:69597:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 2 00000400:00000200:1.0:1788844909.213193:0:69597:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.202.29@tcp(192.168.202.29@tcp)->dst 192.168.202.29@tcp: GET simulate health error: REMOTE_ERROR 00000800:02000400:1.0:1788844909.213196:0:69597:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.202.29: rc = -22 00000800:00000200:1.0:1788844909.213202:0:69597:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [000000008feb9acd] Error -22 on write to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000010:1.0:1788844909.213207:0:69597:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000046d74fc0. 00000400:00000200:1.0:1788844909.213214:0:69597:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788844909.213218:0:69597:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1788844909.213225:0:69597:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5 00000400:00000200:1.0:1788844909.213228:0:69597:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.29@tcp recovery message sent unsuccessfully:-5 00000400:00000200:1.0:1788844909.213233:0:69597:0:(lib-move.c:4104:lnet_handle_recovery_reply()) peer NI (192.168.202.29@tcp) recovery failed with -5 00000400:00000200:2.0:1788844909.798516:0:69893:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:2.0:1788844909.798525:0:69893:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000aa794c31. 00000400:00000200:2.0:1788844909.798535:0:69893:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:2.0:1788844909.798556:0:69893:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000070a783f. 00000400:00000200:2.0:1788844909.798563:0:69893:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:2.0:1788844909.798567:0:69893:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000070a783f. 00000400:00000010:2.0:1788844909.798576:0:69893:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000aa794c31. 00000400:00000200:0.0:1788844910.231950:0:69587:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cbe4bce4 00000400:00000010:0.0:1788844910.231960:0:69587:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000cbe4bce4. 00000400:00000200:0.0:1788844910.231968:0:69587:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0 00000400:00000200:0.0:1788844910.231974:0:69587:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.202.29@tcp recovery ping unlinked 00000400:00000010:0.0:1788844910.231980:0:69587:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000003f067e94. 00000400:00000010:0.0:1788844910.231985:0:69587:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000622eb0be. 00000400:00000200:1.0:1788844911.005718:0:69901:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:1.0:1788844911.005727:0:69901:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000d62b204e. 00000400:00000200:1.0:1788844911.005749:0:69901:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:1.0:1788844911.005768:0:69901:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 000000009e60f96b. 00000400:00000200:1.0:1788844911.005775:0:69901:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:1.0:1788844911.007217:0:69901:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 000000009e60f96b. 00000400:00000010:1.0:1788844911.007237:0:69901:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000d62b204e. 00000400:00000200:2.0:1788844912.129525:0:69909:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:2.0:1788844912.129535:0:69909:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000ebd2e1d2. 00000400:00000200:2.0:1788844912.129546:0:69909:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:2.0:1788844912.129564:0:69909:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000866b5412. 00000400:00000200:2.0:1788844912.129572:0:69909:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:2.0:1788844912.129576:0:69909:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000866b5412. 00000400:00000010:2.0:1788844912.129585:0:69909:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000ebd2e1d2. 00000400:00000010:3.0:1788844912.280382:0:69587:0:(lib-move.c:3721:lnet_recover_peer_nis()) alloc '(ev_info)': 24 at 0000000041b4beef. 00000400:00000010:3.0:1788844912.280395:0:69587:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000007022887f. 00000400:00000010:3.0:1788844912.280402:0:69587:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000005412350. 00000400:00000010:3.0:1788844912.280431:0:69587:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000090802ff9. 00000400:00000200:3.0:1788844912.280439:0:69587:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.29@tcp 00000400:00000200:3.0:1788844912.280464:0:69587:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.29@tcp local destination 00000400:00000200:3.0:1788844912.280471:0:69587:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.29@tcp 00000400:00000200:3.0:1788844912.280476:0:69587:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [24:24:256:4294967295] 192.168.202.29@tcp LPNI seq info [7:7:8:4294967295] 00000400:00000200:3.0:1788844912.280482:0:69587:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:3.0:1788844912.280489:0:69587:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : GET try# 0 00000800:00000200:3.0:1788844912.280495:0:69587:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.29@tcp 00000800:00000010:3.0:1788844912.280501:0:69587:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000008f77e75d. 00000800:00000200:3.0:1788844912.280507:0:69587:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000cd6f6808] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:3.0:1788844912.280515:0:69587:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000200:3.0:1788844912.280520:0:69587:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000008f77e75d type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1788844912.280540:0:69587:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.202.29@tcp added to recovery queue. ping count: 3 next ping: 1484 last alive: 1469 health: 0 00000400:00000200:1.0:1788844912.280604:0:69599:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 3 00000400:00000200:1.0:1788844912.280613:0:69599:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.202.29@tcp(192.168.202.29@tcp)->dst 192.168.202.29@tcp: GET simulate health error: REMOTE_ERROR 00000800:02000400:1.0:1788844912.280617:0:69599:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.202.29: rc = -22 00000800:00000200:1.0:1788844912.280620:0:69599:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [000000006ade1324] Error -22 on write to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000010:1.0:1788844912.280624:0:69599:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000008f77e75d. 00000400:00000200:1.0:1788844912.280640:0:69599:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788844912.280644:0:69599:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1788844912.280651:0:69599:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5 00000400:00000200:1.0:1788844912.280654:0:69599:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.29@tcp recovery message sent unsuccessfully:-5 00000400:00000200:1.0:1788844912.280658:0:69599:0:(lib-move.c:4104:lnet_handle_recovery_reply()) peer NI (192.168.202.29@tcp) recovery failed with -5 00000400:00000200:3.0:1788844913.258678:0:69918:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:3.0:1788844913.258693:0:69918:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000ba5197df. 00000400:00000200:3.0:1788844913.258709:0:69918:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:3.0:1788844913.258732:0:69918:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000c55572a7. 00000400:00000200:3.0:1788844913.258742:0:69918:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:3.0:1788844913.258747:0:69918:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000c55572a7. 00000400:00000010:3.0:1788844913.258762:0:69918:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000ba5197df. 00000400:00000200:0.0:1788844913.311737:0:69587:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000005412350 00000400:00000010:0.0:1788844913.311777:0:69587:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 0000000005412350. 00000400:00000200:0.0:1788844913.311874:0:69587:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0 00000400:00000200:0.0:1788844913.311884:0:69587:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.202.29@tcp recovery ping unlinked 00000400:00000010:0.0:1788844913.311891:0:69587:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000041b4beef. 00000400:00000010:0.0:1788844913.311898:0:69587:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000007022887f. 00000400:00000200:0.0:1788844914.378734:0:69926:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:0.0:1788844914.378742:0:69926:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000f1ea854b. 00000400:00000200:0.0:1788844914.378752:0:69926:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:0.0:1788844914.378768:0:69926:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000526eb36a. 00000400:00000200:0.0:1788844914.378775:0:69926:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:0.0:1788844914.378800:0:69926:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000526eb36a. 00000400:00000010:0.0:1788844914.378809:0:69926:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000f1ea854b. 00000400:00000200:1.0:1788844915.480396:0:69934:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:1.0:1788844915.480409:0:69934:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000763be66e. 00000400:00000200:1.0:1788844915.480438:0:69934:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:1.0:1788844915.480459:0:69934:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000b4ea58d6. 00000400:00000200:1.0:1788844915.480467:0:69934:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:1.0:1788844915.480472:0:69934:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000b4ea58d6. 00000400:00000010:1.0:1788844915.480482:0:69934:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000763be66e. 00000400:00000200:0.0:1788844916.592155:0:69942:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:0.0:1788844916.592168:0:69942:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000003911e19e. 00000400:00000200:0.0:1788844916.592181:0:69942:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:0.0:1788844916.592206:0:69942:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000f1ea854b. 00000400:00000200:0.0:1788844916.592213:0:69942:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:0.0:1788844916.592219:0:69942:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000f1ea854b. 00000400:00000010:0.0:1788844916.592229:0:69942:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000003911e19e. 00000400:00000200:1.0:1788844917.689693:0:69950:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:1.0:1788844917.690457:0:69950:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000763be66e. 00000400:00000200:1.0:1788844917.690472:0:69950:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:1.0:1788844917.690499:0:69950:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000b4ea58d6. 00000400:00000200:1.0:1788844917.690508:0:69950:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:1.0:1788844917.690513:0:69950:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000b4ea58d6. 00000400:00000010:1.0:1788844917.690523:0:69950:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000763be66e. 00000400:00000200:1.0:1788844918.911462:0:69958:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:1.0:1788844918.911471:0:69958:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000763be66e. 00000400:00000200:1.0:1788844918.911486:0:69958:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:1.0:1788844918.911503:0:69958:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000b4ea58d6. 00000400:00000200:1.0:1788844918.911510:0:69958:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:1.0:1788844918.911515:0:69958:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000b4ea58d6. 00000400:00000010:1.0:1788844918.911524:0:69958:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000763be66e. 00000400:00000200:3.0:1788844920.056174:0:69967:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:3.0:1788844920.056189:0:69967:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000508a48de. 00000400:00000200:3.0:1788844920.056200:0:69967:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:3.0:1788844920.056216:0:69967:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000ff2f277f. 00000400:00000200:3.0:1788844920.056223:0:69967:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:3.0:1788844920.056227:0:69967:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000ff2f277f. 00000400:00000010:3.0:1788844920.056237:0:69967:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000508a48de. 00000400:00000010:0.0:1788844920.472178:0:69587:0:(lib-move.c:3721:lnet_recover_peer_nis()) alloc '(ev_info)': 24 at 000000007ddeb37c. 00000400:00000010:0.0:1788844920.472195:0:69587:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000009f97519. 00000400:00000010:0.0:1788844920.472202:0:69587:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cbe4bce4. 00000400:00000010:0.0:1788844920.472212:0:69587:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000004e07275b. 00000400:00000200:0.0:1788844920.472221:0:69587:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.29@tcp 00000400:00000200:0.0:1788844920.472236:0:69587:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.29@tcp local destination 00000400:00000200:0.0:1788844920.472244:0:69587:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.29@tcp 00000400:00000200:0.0:1788844920.472248:0:69587:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [25:25:256:4294967295] 192.168.202.29@tcp LPNI seq info [8:8:8:4294967295] 00000400:00000200:0.0:1788844920.472256:0:69587:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:0.0:1788844920.472264:0:69587:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : GET try# 0 00000800:00000200:0.0:1788844920.472271:0:69587:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.29@tcp 00000800:00000010:0.0:1788844920.472277:0:69587:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000b64fbffd. 00000800:00000200:0.0:1788844920.472284:0:69587:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000cd6f6808] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:0.0:1788844920.472312:0:69587:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000200:0.0:1788844920.472317:0:69587:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000b64fbffd type 2, nob 96 niov 1 nkiov 0 00000400:00000200:0.0:1788844920.472348:0:69587:0:(peer.c:4749:lnet_peer_ni_add_to_recoveryq_locked()) lpni 192.168.202.29@tcp aged out last alive 1469 00000400:00000200:2.0:1788844920.472566:0:69598:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 4 00000400:00000200:2.0:1788844920.472572:0:69598:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.202.29@tcp(192.168.202.29@tcp)->dst 192.168.202.29@tcp: GET simulate health error: REMOTE_ERROR 00000800:02000400:2.0:1788844920.472575:0:69598:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.202.29: rc = -22 00000800:00000200:2.0:1788844920.472580:0:69598:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [000000008feb9acd] Error -22 on write to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000010:2.0:1788844920.472591:0:69598:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000b64fbffd. 00000400:00000200:2.0:1788844920.472597:0:69598:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788844920.472601:0:69598:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1788844920.472608:0:69598:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5 00000400:00000200:2.0:1788844920.472610:0:69598:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.29@tcp recovery message sent unsuccessfully:-5 00000400:00000200:2.0:1788844920.472614:0:69598:0:(lib-move.c:4104:lnet_handle_recovery_reply()) peer NI (192.168.202.29@tcp) recovery failed with -5 00000400:00000200:1.0:1788844921.159357:0:69975:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:1.0:1788844921.159376:0:69975:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000fe82a9de. 00000400:00000010:1.0:1788844921.159383:0:69975:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000fe82a9de. 00000001:02000400:3.0:1788844921.196199:0:69977:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p 00000400:00000200:1.0:1788844921.236723:0:69978:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:1.0:1788844921.236732:0:69978:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000fe82a9de. 00000400:00000010:1.0:1788844921.236739:0:69978:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000fe82a9de. 00000400:00000010:1.0:1788844921.303991:0:69984:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 0000000011597222. 00000400:00000010:1.0:1788844921.304080:0:69984:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 0000000011597222. 00000400:00000010:3.0:1788844921.354635:0:69989:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 0000000096bb4e1a. 00000400:00000010:3.0:1788844921.354723:0:69989:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 0000000096bb4e1a. 00000001:02000400:0.0:1788844921.384571:0:69992:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop del -a 00000400:00000200:1.0:1788844921.415075:0:69993:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844921.415086:0:69993:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844921.415091:0:69993:0:(api-ni.c:3207:nla_strnid()) -> 00000400:00000200:1.0:1788844921.415093:0:69993:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:1.0:1788844921.415095:0:69993:0:(api-ni.c:3207:nla_strnid()) -> 00000400:00000200:1.0:1788844921.415099:0:69993:0:(net_fault.c:254:lnet_drop_rule_del()) src dst 00000400:00000200:1.0:1788844921.415107:0:69993:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp->dst: 255.255.255.255@tcp (1/1, 0) 00000400:00000010:1.0:1788844921.415110:0:69993:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 0000000086baf2f1. 00000400:00000200:1.0:1788844921.415115:0:69993:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp1->dst: 255.255.255.255@tcp1 (1/1, 0) 00000400:00000010:1.0:1788844921.415117:0:69993:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 00000000141ad680. 00000400:00000200:1.0:1788844921.415120:0:69993:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp->dst: 255.255.255.255@tcp (1/1, 0) 00000400:00000010:1.0:1788844921.415122:0:69993:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 00000000f48013a0. 00000400:00000200:1.0:1788844921.415125:0:69993:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp1->dst: 255.255.255.255@tcp1 (1/1, 0) 00000400:00000010:1.0:1788844921.415127:0:69993:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 000000009a9f514e. 00000001:02000400:2.0:1788844921.431551:0:69994:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 0 00000001:02000400:0.0:1788844921.468372:0:69996:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --nid 192.168.202.29@tcp --health 500 00000400:00000200:1.0:1788844921.532149:0:69997:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.202.29@tcp added to recovery queue. ping count: 0 next ping: 1486 last alive: 1469 health: 500 00000400:00000200:2.0:1788844921.653460:0:70003:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:2.0:1788844921.653480:0:70003:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000002979f328. 00000400:00000200:2.0:1788844921.653492:0:70003:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:2.0:1788844921.653512:0:70003:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 0000000016262800. 00000400:00000200:2.0:1788844921.653519:0:70003:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:2.0:1788844921.653522:0:70003:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 0000000016262800. 00000400:00000010:2.0:1788844921.653530:0:70003:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000002979f328. 00000001:02000400:1.0:1788844921.727636:0:70005:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p 00000400:00000200:3.0:1788844921.759907:0:70006:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:3.0:1788844921.759916:0:70006:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000006c80d182. 00000400:00000200:3.0:1788844921.759927:0:70006:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:3.0:1788844921.759946:0:70006:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 0000000099fb3615. 00000400:00000200:3.0:1788844921.759953:0:70006:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:3.0:1788844921.759957:0:70006:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 0000000099fb3615. 00000400:00000010:3.0:1788844921.759967:0:70006:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000006c80d182. 00000400:00000200:1.0:1788844921.839559:0:70012:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:1.0:1788844921.839569:0:70012:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000fe82a9de. 00000400:00000200:1.0:1788844921.839580:0:70012:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:1.0:1788844921.839603:0:70012:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000408d6315. 00000400:00000200:1.0:1788844921.839610:0:70012:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:1.0:1788844921.839614:0:70012:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000408d6315. 00000400:00000010:1.0:1788844921.839621:0:70012:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000fe82a9de. 00000400:00000200:2.0:1788844922.521003:0:69587:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cbe4bce4 00000400:00000010:2.0:1788844922.521013:0:69587:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000cbe4bce4. 00000400:00000200:2.0:1788844922.521021:0:69587:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0 00000400:00000200:2.0:1788844922.521028:0:69587:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.202.29@tcp recovery ping unlinked 00000400:00000010:2.0:1788844922.521034:0:69587:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000007ddeb37c. 00000400:00000010:2.0:1788844922.521038:0:69587:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000009f97519. 00000400:00000010:2.0:1788844922.521047:0:69587:0:(lib-move.c:3721:lnet_recover_peer_nis()) alloc '(ev_info)': 24 at 00000000f598b7c7. 00000400:00000010:2.0:1788844922.521053:0:69587:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006b4976aa. 00000400:00000010:2.0:1788844922.521078:0:69587:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b0945e5a. 00000400:00000010:2.0:1788844922.521098:0:69587:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000004d870181. 00000400:00000200:2.0:1788844922.521103:0:69587:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.29@tcp 00000400:00000200:2.0:1788844922.521114:0:69587:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.29@tcp local destination 00000400:00000200:2.0:1788844922.521121:0:69587:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.29@tcp 00000400:00000200:2.0:1788844922.521124:0:69587:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [26:26:256:4294967295] 192.168.202.29@tcp LPNI seq info [9:9:8:4294967295] 00000400:00000200:2.0:1788844922.521132:0:69587:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:2.0:1788844922.521139:0:69587:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : GET try# 0 00000800:00000200:2.0:1788844922.521145:0:69587:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.29@tcp 00000800:00000010:2.0:1788844922.521155:0:69587:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000007ace9e47. 00000800:00000200:2.0:1788844922.521161:0:69587:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000cd6f6808] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:2.0:1788844922.521169:0:69587:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000200:2.0:1788844922.521173:0:69587:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000007ace9e47 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:2.0:1788844922.521202:0:69587:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.202.29@tcp added to recovery queue. ping count: 1 next ping: 1488 last alive: 1469 health: 500 00000800:00000200:1.0:1788844922.521533:0:69597:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1788844922.521538:0:69597:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000007ace9e47. 00000400:00000200:1.0:1788844922.521542:0:69597:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1788844922.521547:0:69597:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: GET: OK 00000400:00000200:1.0:1788844922.521552:0:69597:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:1.0:1788844922.521554:0:69597:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.29@tcp recovery message sent successfully:0 00000400:00000200:1.0:1788844922.521581:0:69597:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : GET - for me 00000400:00000200:1.0:1788844922.521590:0:69597:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.29@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1788844922.521597:0:69597:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.29@tcp of length 64/272 into md 0xd [1] + 0 00000400:00000200:1.0:1788844922.521606:0:69597:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.29@tcp to MR: 192.168.202.29@tcp local destination 00000400:00000200:1.0:1788844922.521611:0:69597:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [27:27:256:4294967295] 192.168.202.29@tcp LPNI seq info [10:10:8:4294967295] 00000400:00000200:1.0:1788844922.521619:0:69597:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) 192.168.202.29@tcp : REPLY try# 0 00000800:00000200:1.0:1788844922.521624:0:69597:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.202.29@tcp 00000800:00000010:1.0:1788844922.521628:0:69597:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000b810af09. 00000800:00000200:1.0:1788844922.521632:0:69597:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000cd6f6808] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:1.0:1788844922.521636:0:69597:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000200:1.0:1788844922.521639:0:69597:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000b810af09 type 3, nob 160 niov 1 nkiov 1 00000800:00000200:2.0:1788844922.532078:0:69599:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788844922.532086:0:69599:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000b810af09. 00000400:00000200:2.0:1788844922.532095:0:69599:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788844922.532104:0:69599:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: REPLY: OK 00000400:00000200:2.0:1788844922.532139:0:69599:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : REPLY - for me 00000400:00000200:2.0:1788844922.532155:0:69599:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.29@tcp: Reply from 12345-192.168.202.29@tcp of length 64/64 into md 0x2d 00000400:00000200:2.0:1788844922.532167:0:69599:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788844922.532171:0:69599:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: REPLY: OK 00000400:00000200:2.0:1788844922.532175:0:69599:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b0945e5a 00000400:00000010:2.0:1788844922.532177:0:69599:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000b0945e5a. 00000400:00000200:2.0:1788844922.532181:0:69599:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1788844922.532185:0:69599:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000f598b7c7. 00000400:00000010:2.0:1788844922.532188:0:69599:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006b4976aa. 00000400:00000200:0.0:1788844922.956777:0:70020:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:0.0:1788844922.957358:0:70020:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000af6c91a9. 00000400:00000200:0.0:1788844922.957370:0:70020:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:0.0:1788844922.957391:0:70020:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000a1f304c6. 00000400:00000200:0.0:1788844922.957398:0:70020:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:0.0:1788844922.957402:0:70020:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000a1f304c6. 00000400:00000010:0.0:1788844922.957411:0:70020:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000af6c91a9. 00000400:00000200:2.0:1788844924.111508:0:70028:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:2.0:1788844924.111516:0:70028:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000002979f328. 00000400:00000200:2.0:1788844924.111527:0:70028:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:2.0:1788844924.111546:0:70028:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 0000000016262800. 00000400:00000200:2.0:1788844924.111553:0:70028:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:2.0:1788844924.111558:0:70028:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 0000000016262800. 00000400:00000010:2.0:1788844924.111566:0:70028:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000002979f328. 00000400:00000010:2.0:1788844924.574880:0:69587:0:(lib-move.c:3721:lnet_recover_peer_nis()) alloc '(ev_info)': 24 at 00000000bb78fac6. 00000400:00000010:2.0:1788844924.574896:0:69587:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000038c91132. 00000400:00000010:2.0:1788844924.574902:0:69587:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b0945e5a. 00000400:00000010:2.0:1788844924.574911:0:69587:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000071da82e7. 00000400:00000200:2.0:1788844924.574919:0:69587:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.29@tcp 00000400:00000200:2.0:1788844924.574933:0:69587:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.29@tcp local destination 00000400:00000200:2.0:1788844924.574969:0:69587:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.29@tcp 00000400:00000200:2.0:1788844924.574973:0:69587:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [28:28:256:4294967295] 192.168.202.29@tcp LPNI seq info [11:11:8:4294967295] 00000400:00000200:2.0:1788844924.575017:0:69587:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:2.0:1788844924.575024:0:69587:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : GET try# 0 00000800:00000200:2.0:1788844924.575030:0:69587:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.29@tcp 00000800:00000010:2.0:1788844924.575035:0:69587:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f3337948. 00000800:00000200:2.0:1788844924.575063:0:69587:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000cd6f6808] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:2.0:1788844924.575072:0:69587:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000200:2.0:1788844924.575076:0:69587:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f3337948 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:2.0:1788844924.575103:0:69587:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.202.29@tcp added to recovery queue. ping count: 1 next ping: 1490 last alive: 1486 health: 700 00000800:00000200:1.0:1788844924.575397:0:69599:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1788844924.575401:0:69599:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000f3337948. 00000400:00000200:1.0:1788844924.575409:0:69599:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1788844924.575414:0:69599:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: GET: OK 00000400:00000200:1.0:1788844924.575419:0:69599:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:1.0:1788844924.575422:0:69599:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.29@tcp recovery message sent successfully:0 00000400:00000200:1.0:1788844924.575459:0:69599:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : GET - for me 00000400:00000200:1.0:1788844924.575469:0:69599:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.29@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1788844924.575484:0:69599:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.29@tcp of length 64/272 into md 0xd [1] + 0 00000400:00000200:1.0:1788844924.575493:0:69599:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.29@tcp to MR: 192.168.202.29@tcp local destination 00000400:00000200:1.0:1788844924.575497:0:69599:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [29:29:256:4294967295] 192.168.202.29@tcp LPNI seq info [12:12:8:4294967295] 00000400:00000200:1.0:1788844924.575504:0:69599:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) 192.168.202.29@tcp : REPLY try# 0 00000800:00000200:1.0:1788844924.575508:0:69599:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.202.29@tcp 00000800:00000010:1.0:1788844924.575512:0:69599:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000007e4415f4. 00000800:00000200:1.0:1788844924.575516:0:69599:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000cd6f6808] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:1.0:1788844924.575520:0:69599:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000200:1.0:1788844924.575523:0:69599:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000007e4415f4 type 3, nob 160 niov 1 nkiov 1 00000800:00000200:1.0:1788844924.575595:0:69599:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1788844924.575598:0:69599:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000007e4415f4. 00000400:00000200:1.0:1788844924.575601:0:69599:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1788844924.575604:0:69599:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: REPLY: OK 00000400:00000200:1.0:1788844924.575619:0:69599:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : REPLY - for me 00000400:00000200:1.0:1788844924.575626:0:69599:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.29@tcp: Reply from 12345-192.168.202.29@tcp of length 64/64 into md 0x31 00000400:00000200:1.0:1788844924.575635:0:69599:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1788844924.575638:0:69599:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: REPLY: OK 00000400:00000200:1.0:1788844924.575642:0:69599:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b0945e5a 00000400:00000010:1.0:1788844924.575644:0:69599:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000b0945e5a. 00000400:00000200:1.0:1788844924.575646:0:69599:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:1.0:1788844924.575649:0:69599:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000bb78fac6. 00000400:00000010:1.0:1788844924.575652:0:69599:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000038c91132. 00000400:00000200:1.0:1788844925.225054:0:70036:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:1.0:1788844925.225067:0:70036:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000e7699694. 00000400:00000200:1.0:1788844925.225081:0:70036:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:1.0:1788844925.225103:0:70036:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000afce11de. 00000400:00000200:1.0:1788844925.225111:0:70036:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:1.0:1788844925.225116:0:70036:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000afce11de. 00000400:00000010:1.0:1788844925.225125:0:70036:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000e7699694. 00000400:00000200:1.0:1788844926.357183:0:70044:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:1.0:1788844926.357196:0:70044:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000e7699694. 00000400:00000200:1.0:1788844926.357210:0:70044:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:1.0:1788844926.357232:0:70044:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000afce11de. 00000400:00000200:1.0:1788844926.357240:0:70044:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:1.0:1788844926.357244:0:70044:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000afce11de. 00000400:00000010:1.0:1788844926.357253:0:70044:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000e7699694. 00000400:00000010:2.0:1788844926.616054:0:69587:0:(lib-move.c:3721:lnet_recover_peer_nis()) alloc '(ev_info)': 24 at 00000000c646dc9e. 00000400:00000010:2.0:1788844926.616077:0:69587:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000fa760abd. 00000400:00000010:2.0:1788844926.616085:0:69587:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000013d53e7c. 00000400:00000010:2.0:1788844926.616096:0:69587:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000008cf8c931. 00000400:00000200:2.0:1788844926.616107:0:69587:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.29@tcp 00000400:00000200:2.0:1788844926.616125:0:69587:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.29@tcp local destination 00000400:00000200:2.0:1788844926.616134:0:69587:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.29@tcp 00000400:00000200:2.0:1788844926.616138:0:69587:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [30:30:256:4294967295] 192.168.202.29@tcp LPNI seq info [13:13:8:4294967295] 00000400:00000200:2.0:1788844926.616147:0:69587:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:2.0:1788844926.616154:0:69587:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : GET try# 0 00000800:00000200:2.0:1788844926.616160:0:69587:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.29@tcp 00000800:00000010:2.0:1788844926.616166:0:69587:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000b8270d8a. 00000800:00000200:2.0:1788844926.616173:0:69587:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000cd6f6808] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:2.0:1788844926.616183:0:69587:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000200:2.0:1788844926.616188:0:69587:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000b8270d8a type 2, nob 96 niov 1 nkiov 0 00000400:00000200:2.0:1788844926.616208:0:69587:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.202.29@tcp added to recovery queue. ping count: 1 next ping: 1492 last alive: 1488 health: 900 00000800:00000200:1.0:1788844926.617253:0:69599:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1788844926.617268:0:69599:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000b8270d8a. 00000400:00000200:1.0:1788844926.617278:0:69599:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1788844926.617285:0:69599:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: GET: OK 00000400:00000200:1.0:1788844926.617292:0:69599:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:1.0:1788844926.617295:0:69599:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.29@tcp recovery message sent successfully:0 00000400:00000200:1.0:1788844926.617340:0:69599:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : GET - for me 00000400:00000200:1.0:1788844926.617353:0:69599:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.29@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1788844926.617363:0:69599:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.29@tcp of length 64/272 into md 0xd [1] + 0 00000400:00000200:1.0:1788844926.617385:0:69599:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.29@tcp to MR: 192.168.202.29@tcp local destination 00000400:00000200:1.0:1788844926.617390:0:69599:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [31:31:256:4294967295] 192.168.202.29@tcp LPNI seq info [14:14:8:4294967295] 00000400:00000200:1.0:1788844926.617398:0:69599:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) 192.168.202.29@tcp : REPLY try# 0 00000800:00000200:1.0:1788844926.617402:0:69599:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.202.29@tcp 00000800:00000010:1.0:1788844926.617406:0:69599:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000007e4415f4. 00000800:00000200:1.0:1788844926.617411:0:69599:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000cd6f6808] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:1.0:1788844926.617417:0:69599:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000200:1.0:1788844926.617420:0:69599:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000007e4415f4 type 3, nob 160 niov 1 nkiov 1 00000800:00000200:2.0:1788844926.637143:0:69597:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788844926.637156:0:69597:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000007e4415f4. 00000400:00000200:2.0:1788844926.637180:0:69597:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788844926.637192:0:69597:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: REPLY: OK 00000400:00000200:2.0:1788844926.637235:0:69597:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : REPLY - for me 00000400:00000200:2.0:1788844926.637272:0:69597:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.29@tcp: Reply from 12345-192.168.202.29@tcp of length 64/64 into md 0x35 00000400:00000200:2.0:1788844926.637288:0:69597:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788844926.637292:0:69597:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: REPLY: OK 00000400:00000200:2.0:1788844926.637297:0:69597:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000013d53e7c 00000400:00000010:2.0:1788844926.637300:0:69597:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 0000000013d53e7c. 00000400:00000200:2.0:1788844926.637304:0:69597:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1788844926.637309:0:69597:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000c646dc9e. 00000400:00000010:2.0:1788844926.637324:0:69597:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000fa760abd. 00000400:00000200:1.0:1788844927.537079:0:70053:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:1.0:1788844927.537093:0:70053:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000afce11de. 00000400:00000200:1.0:1788844927.537107:0:70053:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.202.29@tcp 00000400:00000010:1.0:1788844927.537134:0:70053:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000ecc6ac02. 00000400:00000200:1.0:1788844927.537143:0:70053:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.202.29@tcp 00000400:00000010:1.0:1788844927.537152:0:70053:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000ecc6ac02. 00000400:00000010:1.0:1788844927.537162:0:70053:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000afce11de. 00000400:00000200:1.0:1788844928.704133:0:70061:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:1.0:1788844928.704141:0:70061:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000afce11de. 00000400:00000010:1.0:1788844928.704148:0:70061:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000afce11de. 00000001:02000400:1.0:1788844928.759393:0:70063:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p 00000400:00000200:1.0:1788844928.801139:0:70064:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1 00000400:00000010:1.0:1788844928.801148:0:70064:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000afce11de. 00000400:00000010:1.0:1788844928.801156:0:70064:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000afce11de. 00000001:02000400:2.0:1788844928.880660:0:70067:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure 00000400:00000200:3.0:1788844928.914672:0:70068:0:(net_fault.c:254:lnet_drop_rule_del()) src dst 00000400:00000200:2.0:1788844928.920074:0:69588:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: -108 00000400:00000200:2.0:1788844928.920083:0:69588:0:(peer.c:4385:lnet_peer_discovery()) stopping 00000400:00000200:2.0:1788844928.920087:0:69588:0:(peer.c:4431:lnet_peer_discovery()) stopped 00000400:00000200:3.0:1788844928.920364:0:70068:0:(peer.c:4488:lnet_peer_discovery_stop()) discovery stopped 00000400:00000010:3.0:1788844928.920750:0:70068:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 00000000d49c0485. 00000400:00000010:3.0:1788844928.920759:0:70068:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 000000007bd1e39d. 00000400:00000010:3.0:1788844928.920761:0:70068:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 00000000d0711c20. 00000400:00000010:3.0:1788844928.920763:0:70068:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 0000000034fcdbe8. 00000400:00000010:3.0:1788844928.920765:0:70068:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 0000000090802ff9. 00000400:00000010:3.0:1788844928.920767:0:70068:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 000000004e07275b. 00000400:00000010:3.0:1788844928.920769:0:70068:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 000000004d870181. 00000400:00000010:3.0:1788844928.920771:0:70068:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 0000000071da82e7. 00000400:00000010:3.0:1788844928.920773:0:70068:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 000000008cf8c931. 00000400:00000010:3.0:1788844928.920776:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000d89c206e. 00000400:00000010:3.0:1788844928.920808:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000004f5dfc7e. 00000400:00000010:3.0:1788844928.920813:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000e31e2148. 00000400:00000010:3.0:1788844928.920814:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000003cc5a3ad. 00000400:00000010:3.0:1788844928.920822:0:70068:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d032620a. 00000400:00000200:3.0:1788844928.920826:0:70068:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000001ef93f4 00000400:00000010:3.0:1788844928.920828:0:70068:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 0000000001ef93f4. 00000400:00000200:3.0:1788844928.920850:0:70068:0:(api-ni.c:2393:lnet_push_target_event_handler()) type 6 status 0 unlinked 1 00000400:00000010:3.0:1788844928.920854:0:70068:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000009eeb148. 00000400:00000400:3.0:1788844928.920859:0:70068:0:(lib-ptl.c:964:lnet_clear_lazy_portal()) Active lazy portal 0 on exit 00000400:00000010:3.0:1788844928.928513:0:70068:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000007d2552db. 00000400:00000200:3.0:1788844928.928526:0:70068:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e8f88037 00000400:00000010:3.0:1788844928.928528:0:70068:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000e8f88037. 00000400:00000010:3.0:1788844928.928677:0:70068:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 73 at 000000002a657409. 00000400:00000200:2.0:1788844928.928869:0:69600:0:(acceptor.c:659:lnet_acceptor()) Acceptor stopping 00000400:00000200:3.0:1788844928.929022:0:70068:0:(router.c:792:lnet_del_route()) Del route: net <255:65535> : gw 00000400:00000200:3.0:1788844928.929046:0:70068:0:(lo.c:49:lolnd_shutdown()) shutdown 00000400:00000010:3.0:1788844928.929049:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000ab0608cd. 00000400:00000010:3.0:1788844928.929054:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000024757873. 00000400:00000010:3.0:1788844928.929057:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000001188061. 00000400:00000010:3.0:1788844928.929058:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000058f22f4a. 00000400:00000010:3.0:1788844928.929061:0:70068:0:(config.c:293:lnet_ni_free()) kfreed 'ni': 560 at 000000004c36459b. 00000400:00000010:3.0:1788844928.929071:0:70068:0:(config.c:329:lnet_net_free()) kfreed 'net': 176 at 000000005d80f1e2. 00000800:00000200:3.1:1788844928.929097:0:70068:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 2, ksnr_connected a ksnr_max_conns 1 00000800:00000200:3.1:1788844928.929104:0:70068:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 3, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:3.1:1788844928.929106:0:70068:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:3.1:1788844928.929107:0:70068:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 1, ksnr_connected 0 ksnr_max_conns 1 00000800:00000010:3.1:1788844928.929109:0:70068:0:(socklnd.c:126:ksocknal_destroy_conn_cb()) kfreed 'conn_cb': 192 at 00000000e735332f. 00000400:00020000:3.0:1788844928.929115:0:70068:0:(acceptor.c:252:lnet_acceptor_remove_socket()) Interface ens2 not found 00000800:00000200:1.0:1788844928.930026:0:69596:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 00000000a5ec57bb 00000800:00000010:1.0:1788844928.930032:0:69596:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 00000000a5ec57bb. 00000800:00000200:1.0:1788844928.930035:0:69596:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 000000006ade1324 00000800:00000010:1.0:1788844928.930036:0:69596:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 000000006ade1324. 00000800:00000200:1.0:1788844928.930062:0:69596:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 0000000071fab5be 00000800:00000010:1.0:1788844928.930064:0:69596:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 0000000071fab5be. 00000800:00000200:1.0:1788844928.930067:0:69596:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 000000008feb9acd 00000800:00000200:1.0:1788844928.930075:0:69596:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.202.29@tcp 00000000cd6f6808 deleted 00000800:00000010:1.0:1788844928.930077:0:69596:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 00000000cd6f6808. 00000800:00000010:1.0:1788844928.930093:0:69596:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 000000008feb9acd. 00000800:00000010:1.0:1788844928.953260:0:70068:0:(socklnd.c:2498:ksocknal_shutdown()) kfreed 'net': 208 at 000000005620bd8e. 00000400:02000000:1.0:1788844928.953282:0:70068:0:(api-ni.c:2563:lnet_clear_zombies_nis_locked()) Removed LNI 192.168.202.29@tcp 00000400:00000010:1.0:1788844928.955067:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000a1603e8c. 00000400:00000010:1.0:1788844928.955078:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000007078147c. 00000400:00000010:1.0:1788844928.955081:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000266a8074. 00000400:00000010:1.0:1788844928.955083:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000003feecb1d. 00000400:00000010:1.0:1788844928.955086:0:70068:0:(config.c:286:lnet_ni_free()) kfreed 'ni->ni_interface': 5 at 000000007d7f717e. 00000400:00000010:1.0:1788844928.955089:0:70068:0:(config.c:293:lnet_ni_free()) kfreed 'ni': 560 at 00000000c7a6e37a. 00000400:00000200:1.0:1788844928.955107:0:70068:0:(peer.c:620:lnet_peer_ni_clr_local_net_locked()) 0000000061c20f74 nid 192.168.202.29@tcp detached from local net 00000400:00000010:1.0:1788844928.955112:0:70068:0:(config.c:329:lnet_net_free()) kfreed 'net': 176 at 00000000255b2d0f. 00000400:00020000:1.0:1788844928.955137:0:70068:0:(acceptor.c:252:lnet_acceptor_remove_socket()) Interface ens2 not found 00000800:00000010:1.0:1788844928.955140:0:70068:0:(socklnd.c:2498:ksocknal_shutdown()) kfreed 'net': 208 at 0000000085b28da0. 00000800:00000010:1.0:1788844928.955143:0:70068:0:(socklnd.c:2235:ksocknal_base_shutdown()) before NAL cleanup: kmem 418797 00000800:00000200:1.0:1788844928.955153:0:70068:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_GOING_DOWN, iface=lo ifindex 1 state 0 ns 000000004dc0f701 00000800:00000200:1.0:1788844928.955175:0:70068:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_DOWN, iface=lo ifindex 1 state 0 ns 000000004dc0f701 00000800:00000200:1.0:1788844928.955177:0:70068:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UNREGISTER, iface=lo ifindex 1 state 0 ns 000000004dc0f701 00000800:00000200:1.0:1788844928.955179:0:70068:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_GOING_DOWN, iface=ens2 ifindex 2 state 6 ns 000000004dc0f701 00000800:00000200:1.0:1788844928.955181:0:70068:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_DOWN, iface=ens2 ifindex 2 state 6 ns 000000004dc0f701 00000800:00000200:1.0:1788844928.955183:0:70068:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UNREGISTER, iface=ens2 ifindex 2 state 6 ns 000000004dc0f701 00000400:00000010:3.0:1788844928.995975:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 0000000029a4ebe6. 00000400:00000010:3.0:1788844928.995987:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000051e05e20. 00000800:00000010:3.0:1788844928.995990:0:70068:0:(socklnd.c:2296:ksocknal_base_shutdown()) after NAL cleanup: kmem 418645 00000400:02000000:0.0:1788844929.000362:0:70068:0:(api-ni.c:2563:lnet_clear_zombies_nis_locked()) Removed LNI 192.168.202.29@tcp1 00000400:00000010:0.0:1788844929.010674:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000065964dfc. 00000400:00000010:0.0:1788844929.010684:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000022b887fa. 00000400:00000010:0.0:1788844929.010686:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000004a4f5b5d. 00000400:00000010:0.0:1788844929.010688:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000000c293bbb. 00000400:00000010:0.0:1788844929.010690:0:70068:0:(config.c:286:lnet_ni_free()) kfreed 'ni->ni_interface': 5 at 0000000035d83818. 00000400:00000010:0.0:1788844929.010693:0:70068:0:(config.c:293:lnet_ni_free()) kfreed 'ni': 560 at 000000002d7762e0. 00000400:00000200:0.0:1788844929.010709:0:70068:0:(peer.c:442:lnet_peer_del_locked()) peer 192.168.202.29@tcp1 00000400:00000200:0.0:1788844929.010714:0:70068:0:(peer.c:364:lnet_peer_detach_peer_ni_locked()) peer 192.168.202.29@tcp1 NID 192.168.202.29@tcp 00000400:00000200:0.0:1788844929.010716:0:70068:0:(peer.c:2392:lnet_destroy_peer_ni_locked()) 0000000061c20f74 nid 192.168.202.29@tcp 00000400:00000010:0.0:1788844929.010719:0:70068:0:(peer.c:2422:lnet_destroy_peer_ni_locked()) kfreed 'lpni': 480 at 0000000061c20f74. 00000400:00000200:0.0:1788844929.010722:0:70068:0:(peer.c:213:lnet_destroy_peer_net_locked()) 0000000080339687 net tcp 00000400:00000010:0.0:1788844929.010723:0:70068:0:(peer.c:220:lnet_destroy_peer_net_locked()) kfreed 'lpn': 72 at 0000000080339687. 00000400:00000200:0.0:1788844929.010727:0:70068:0:(peer.c:364:lnet_peer_detach_peer_ni_locked()) peer 192.168.202.29@tcp1 NID 192.168.202.29@tcp1 00000400:00000200:0.0:1788844929.010728:0:70068:0:(peer.c:2392:lnet_destroy_peer_ni_locked()) 000000000d8bd57e nid 192.168.202.29@tcp1 00000400:00000010:0.0:1788844929.010730:0:70068:0:(peer.c:2422:lnet_destroy_peer_ni_locked()) kfreed 'lpni': 480 at 000000000d8bd57e. 00000400:00000200:0.0:1788844929.010732:0:70068:0:(peer.c:213:lnet_destroy_peer_net_locked()) 00000000856394e9 net tcp1 00000400:00000010:0.0:1788844929.010733:0:70068:0:(peer.c:220:lnet_destroy_peer_net_locked()) kfreed 'lpn': 72 at 00000000856394e9. 00000400:00000200:0.0:1788844929.010736:0:70068:0:(peer.c:274:lnet_destroy_peer_locked()) 00000000c22a5afc nid 192.168.202.29@tcp1 00000400:00000010:0.0:1788844929.010758:0:70068:0:(peer.c:304:lnet_destroy_peer_locked()) kfreed 'lp': 360 at 00000000c22a5afc. 00000400:00000010:0.0:1788844929.013621:0:70068:0:(config.c:329:lnet_net_free()) kfreed 'net': 176 at 000000000f9bd944. 00000400:00000010:0.0:1788844929.013635:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000009a7ce84e. 00000400:00000010:0.0:1788844929.013639:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000007d40e20f. 00000400:00000010:0.0:1788844929.013645:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000005636bcd6. 00000400:00000010:0.0:1788844929.013648:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000fc0cfe08. 00000400:00000010:0.0:1788844929.013650:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000406ca20f. 00000400:00000010:0.0:1788844929.013652:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000065731974. 00000400:00000010:0.0:1788844929.013657:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000595b4ee4. 00000400:00000010:0.0:1788844929.013659:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000003f664474. 00000400:00000010:0.0:1788844929.013660:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000031b7289f. 00000400:00000010:0.0:1788844929.013662:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000004c8a6973. 00000400:00000010:0.0:1788844929.013665:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000019bdd1d9. 00000400:00000010:0.0:1788844929.013667:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000c984575c. 00000400:00000010:0.0:1788844929.013669:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000000de8427e. 00000400:00000010:0.0:1788844929.013670:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000004096d1cf. 00000400:00000010:0.0:1788844929.013673:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000003abcc076. 00000400:00000010:0.0:1788844929.013676:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000009a5490b5. 00000400:00000010:0.0:1788844929.013677:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000d9819b09. 00000400:00000010:0.0:1788844929.013679:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000fed799a0. 00000400:00000010:0.0:1788844929.013683:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000cce1b85e. 00000400:00000010:0.0:1788844929.013685:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000a60edc7f. 00000400:00000010:0.0:1788844929.013687:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000e895244b. 00000400:00000010:0.0:1788844929.013688:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000067c24a58. 00000400:00000010:0.0:1788844929.013691:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f8b96da2. 00000400:00000010:0.0:1788844929.013694:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000083415f8f. 00000400:00000010:0.0:1788844929.013695:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000060c544f. 00000400:00000010:0.0:1788844929.013697:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e14976b5. 00000400:00000010:0.0:1788844929.013700:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000002bbfaf7b. 00000400:00000010:0.0:1788844929.013702:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000818f8a99. 00000400:00000010:0.0:1788844929.013704:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000fe1c54f1. 00000400:00000010:0.0:1788844929.013705:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e0634bff. 00000400:00000010:0.0:1788844929.013708:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000001f03eb7b. 00000400:00000010:0.0:1788844929.013722:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000bf7845de. 00000400:00000010:0.0:1788844929.013723:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000009e80ec08. 00000400:00000010:0.0:1788844929.013725:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000009141af7. 00000400:00000010:0.0:1788844929.013729:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f8577e7d. 00000400:00000010:0.0:1788844929.013731:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000005b48385f. 00000400:00000010:0.0:1788844929.013733:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000c45bae3a. 00000400:00000010:0.0:1788844929.013734:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000003bd6e968. 00000400:00000010:0.0:1788844929.013737:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000625a9453. 00000400:00000010:0.0:1788844929.013739:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000064ba31c8. 00000400:00000010:0.0:1788844929.013741:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000796d72dc. 00000400:00000010:0.0:1788844929.013742:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000bae90281. 00000400:00000010:0.0:1788844929.013745:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000008a53d52. 00000400:00000010:0.0:1788844929.013747:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000085b9359. 00000400:00000010:0.0:1788844929.013749:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000007e128e28. 00000400:00000010:0.0:1788844929.013750:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000ce84b06d. 00000400:00000010:0.0:1788844929.013753:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000dd0c6b59. 00000400:00000010:0.0:1788844929.013756:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000004d273e1. 00000400:00000010:0.0:1788844929.013758:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000001da49bc8. 00000400:00000010:0.0:1788844929.013759:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000aef65602. 00000400:00000010:0.0:1788844929.013762:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000026a45d92. 00000400:00000010:0.0:1788844929.013764:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000ddbf54cc. 00000400:00000010:0.0:1788844929.013766:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000faefeea4. 00000400:00000010:0.0:1788844929.013767:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000008c06e919. 00000400:00000010:0.0:1788844929.013770:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000365db3e4. 00000400:00000010:0.0:1788844929.013772:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000003be6f71. 00000400:00000010:0.0:1788844929.013774:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000442dadfb. 00000400:00000010:0.0:1788844929.013775:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000085c4398b. 00000400:00000010:0.0:1788844929.013809:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ce632e44. 00000400:00000010:0.0:1788844929.013813:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000002779f1ce. 00000400:00000010:0.0:1788844929.013815:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000001616a2af. 00000400:00000010:0.0:1788844929.013816:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000007a8ccd8b. 00000400:00000010:0.0:1788844929.013819:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000001dc6f449. 00000400:00000010:0.0:1788844929.013823:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000a83e7e57. 00000400:00000010:0.0:1788844929.013830:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000e61a5bde. 00000400:00000010:0.0:1788844929.013832:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000028431340. 00000400:00000010:0.0:1788844929.013836:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f2896859. 00000400:00000010:0.0:1788844929.013837:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000248a6efb. 00000400:00000010:0.0:1788844929.013839:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000023abbb72. 00000400:00000010:0.0:1788844929.013841:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000afab913f. 00000400:00000010:0.0:1788844929.013844:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000666acba5. 00000400:00000010:0.0:1788844929.013846:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000080acbd77. 00000400:00000010:0.0:1788844929.013848:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000417090ac. 00000400:00000010:0.0:1788844929.013850:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000049403b1d. 00000400:00000010:0.0:1788844929.013853:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000085d84fe0. 00000400:00000010:0.0:1788844929.013855:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000005450bc93. 00000400:00000010:0.0:1788844929.013856:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000005c86a724. 00000400:00000010:0.0:1788844929.013858:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000ba313706. 00000400:00000010:0.0:1788844929.013861:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ac6fdf8f. 00000400:00000010:0.0:1788844929.013863:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000386ff875. 00000400:00000010:0.0:1788844929.013864:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000406cc200. 00000400:00000010:0.0:1788844929.013866:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000002f400dfa. 00000400:00000010:0.0:1788844929.013869:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000b075f539. 00000400:00000010:0.0:1788844929.013871:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000e59582d5. 00000400:00000010:0.0:1788844929.013873:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000add70cbc. 00000400:00000010:0.0:1788844929.013874:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000ca35aead. 00000400:00000010:0.0:1788844929.013877:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000091cba7af. 00000400:00000010:0.0:1788844929.013879:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000005399dc9. 00000400:00000010:0.0:1788844929.013880:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000aea4744f. 00000400:00000010:0.0:1788844929.013881:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000016f2e852. 00000400:00000010:0.0:1788844929.013884:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000041fda1eb. 00000400:00000010:0.0:1788844929.013887:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000084a141da. 00000400:00000010:0.0:1788844929.013888:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000d11d8cd3. 00000400:00000010:0.0:1788844929.013890:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e627d6a9. 00000400:00000010:0.0:1788844929.013893:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000006051c759. 00000400:00000010:0.0:1788844929.013895:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000bc812c2a. 00000400:00000010:0.0:1788844929.013897:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000003e9b1fb4. 00000400:00000010:0.0:1788844929.013905:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e624a748. 00000400:00000010:0.0:1788844929.013908:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000003c7946d5. 00000400:00000010:0.0:1788844929.013910:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000003a74fcc1. 00000400:00000010:0.0:1788844929.013911:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000003f54b82f. 00000400:00000010:0.0:1788844929.013913:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000082c3d7ae. 00000400:00000010:0.0:1788844929.013915:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000049f837f7. 00000400:00000010:0.0:1788844929.013917:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000008334e28c. 00000400:00000010:0.0:1788844929.013919:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002f2618a5. 00000400:00000010:0.0:1788844929.013920:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000406afaf8. 00000400:00000010:0.0:1788844929.013923:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000004c570512. 00000400:00000010:0.0:1788844929.013924:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000217cf18f. 00000400:00000010:0.0:1788844929.013926:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000fb507bf3. 00000400:00000010:0.0:1788844929.013927:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000fc8d4ff3. 00000400:00000010:0.0:1788844929.013930:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000b555330b. 00000400:00000010:0.0:1788844929.013932:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000be5e9941. 00000400:00000010:0.0:1788844929.013934:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000b6e0beba. 00000400:00000010:0.0:1788844929.013935:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000624d8cba. 00000400:00000010:0.0:1788844929.013937:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000dc206ed9. 00000400:00000010:0.0:1788844929.013940:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000dedcd839. 00000400:00000010:0.0:1788844929.013941:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000001ecd7c69. 00000400:00000010:0.0:1788844929.013942:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000aa1008e9. 00000400:00000010:0.0:1788844929.013945:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000d584d457. 00000400:00000010:0.0:1788844929.013947:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000007978d37a. 00000400:00000010:0.0:1788844929.013949:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000005bb316ba. 00000400:00000010:0.0:1788844929.013950:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e64a6c1e. 00000400:00000010:0.0:1788844929.013953:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000d7072bc3. 00000400:00000010:0.0:1788844929.013955:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000659cb8f1. 00000400:00000010:0.0:1788844929.013957:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000086622aa7. 00000400:00000010:0.0:1788844929.013958:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000004ef956bc. 00000400:00000010:0.0:1788844929.013961:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000035c7b992. 00000400:00000010:0.0:1788844929.013964:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000dfdd4354. 00000400:00000010:0.0:1788844929.013966:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000f556e4e2. 00000400:00000010:0.0:1788844929.013967:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000071e56a46. 00000400:00000010:0.0:1788844929.013975:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000005aefee44. 00000400:00000010:0.0:1788844929.013991:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000302c289e. 00000400:00000010:0.0:1788844929.013993:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000043dcb519. 00000400:00000010:0.0:1788844929.013995:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000009d22323b. 00000400:00000010:0.0:1788844929.013998:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000012bb5fad. 00000400:00000010:0.0:1788844929.014000:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000008ed17bca. 00000400:00000010:0.0:1788844929.014001:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000ff961f49. 00000400:00000010:0.0:1788844929.014002:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000b5fc2b56. 00000400:00000010:0.0:1788844929.014005:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000b7b45c03. 00000400:00000010:0.0:1788844929.014007:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000222cbdfd. 00000400:00000010:0.0:1788844929.014008:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000a47d9b88. 00000400:00000010:0.0:1788844929.014010:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000064f0c89d. 00000400:00000010:0.0:1788844929.014013:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000058d5f192. 00000400:00000010:0.0:1788844929.014015:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000bd712de2. 00000400:00000010:0.0:1788844929.014016:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002f33c48b. 00000400:00000010:0.0:1788844929.014017:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000006075400d. 00000400:00000010:0.0:1788844929.014020:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000004b3231fa. 00000400:00000010:0.0:1788844929.014022:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000020b3104b. 00000400:00000010:0.0:1788844929.014024:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000099b9809d. 00000400:00000010:0.0:1788844929.014025:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000f1b77af6. 00000400:00000010:0.0:1788844929.014028:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000c587f8d0. 00000400:00000010:0.0:1788844929.014029:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000001b227634. 00000400:00000010:0.0:1788844929.014031:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000001674e34b. 00000400:00000010:0.0:1788844929.014032:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000052ffac95. 00000400:00000010:0.0:1788844929.014035:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000001ba26530. 00000400:00000010:0.0:1788844929.014037:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000060e4d16. 00000400:00000010:0.0:1788844929.014039:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000dbfb5833. 00000400:00000010:0.0:1788844929.014040:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000000e13b884. 00000400:00000010:0.0:1788844929.014043:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000d8f2cce8. 00000400:00000010:0.0:1788844929.014044:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f0e8c66a. 00000400:00000010:0.0:1788844929.014046:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000000202246d. 00000400:00000010:0.0:1788844929.014047:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000006379be4f. 00000400:00000010:0.0:1788844929.014050:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000e5717462. 00000400:00000010:0.0:1788844929.014057:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000e2d09096. 00000400:00000010:0.0:1788844929.014058:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000fcb9842d. 00000400:00000010:0.0:1788844929.014060:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000076532758. 00000400:00000010:0.0:1788844929.014062:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f0f3900e. 00000400:00000010:0.0:1788844929.014064:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000008cfacad6. 00000400:00000010:0.0:1788844929.014065:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000001d092066. 00000400:00000010:0.0:1788844929.014067:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000001dfc40c1. 00000400:00000010:0.0:1788844929.014070:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000003ba3b00. 00000400:00000010:0.0:1788844929.014072:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000fa63a178. 00000400:00000010:0.0:1788844929.014073:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000015a7eedd. 00000400:00000010:0.0:1788844929.014075:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000afa3651a. 00000400:00000010:0.0:1788844929.014078:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000062b2d79e. 00000400:00000010:0.0:1788844929.014092:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000e8cac664. 00000400:00000010:0.0:1788844929.014094:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000008e1d2ffa. 00000400:00000010:0.0:1788844929.014096:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000034b85e67. 00000400:00000010:0.0:1788844929.014098:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000004950b76d. 00000400:00000010:0.0:1788844929.014102:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000e5b2fa44. 00000400:00000010:0.0:1788844929.014103:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000a5a0fca5. 00000400:00000010:0.0:1788844929.014105:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000dc627611. 00000400:00000010:0.0:1788844929.014108:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000002ca18eb7. 00000400:00000010:0.0:1788844929.014110:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000009ef0139b. 00000400:00000010:0.0:1788844929.014112:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000bd38ab3a. 00000400:00000010:0.0:1788844929.014113:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000092363bc8. 00000400:00000010:0.0:1788844929.014116:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000e492d794. 00000400:00000010:0.0:1788844929.014118:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000002953d5ee. 00000400:00000010:0.0:1788844929.014120:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000003fe01660. 00000400:00000010:0.0:1788844929.014121:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000046fce6dd. 00000400:00000010:0.0:1788844929.014124:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ea9620cd. 00000400:00000010:0.0:1788844929.014126:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000ff1cd0ef. 00000400:00000010:0.0:1788844929.014128:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000ba12b6e3. 00000400:00000010:0.0:1788844929.014129:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000090b4a2cb. 00000400:00000010:0.0:1788844929.014133:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f4841a6c. 00000400:00000010:0.0:1788844929.014134:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000fe307cec. 00000400:00000010:0.0:1788844929.014142:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000022adf9dc. 00000400:00000010:0.0:1788844929.014144:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000003bd47c82. 00000400:00000010:0.0:1788844929.014147:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000e29cd06f. 00000400:00000010:0.0:1788844929.014149:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000007f2c1e88. 00000400:00000010:0.0:1788844929.014151:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000233100cf. 00000400:00000010:0.0:1788844929.014153:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e29ecc97. 00000400:00000010:0.0:1788844929.014156:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000b6393d4e. 00000400:00000010:0.0:1788844929.014172:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000ed069f39. 00000400:00000010:0.0:1788844929.014175:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000d68be744. 00000400:00000010:0.0:1788844929.014176:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000052a64fe4. 00000400:00000010:0.0:1788844929.014179:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000004d403923. 00000400:00000010:0.0:1788844929.014181:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000007b0c6120. 00000400:00000010:0.0:1788844929.014182:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000005f64e2c. 00000400:00000010:0.0:1788844929.014184:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000907bc77a. 00000400:00000010:0.0:1788844929.014187:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000094e9313b. 00000400:00000010:0.0:1788844929.014189:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000000b0cf052. 00000400:00000010:0.0:1788844929.014190:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000fb207986. 00000400:00000010:0.0:1788844929.014193:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000292c3b3c. 00000400:00000010:0.0:1788844929.014195:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000090d28e3a. 00000400:00000010:0.0:1788844929.014198:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000002d892f93. 00000400:00000010:0.0:1788844929.014200:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000009486a532. 00000400:00000010:0.0:1788844929.014202:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000007fb8397b. 00000400:00000010:0.0:1788844929.014205:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000b06228b2. 00000400:00000010:0.0:1788844929.014207:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000803935c1. 00000400:00000010:0.0:1788844929.014209:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000013c3a0f2. 00000400:00000010:0.0:1788844929.014210:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000f47d3b45. 00000400:00000010:0.0:1788844929.014215:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000011c2bc27. 00000400:00000010:0.0:1788844929.014217:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000614fdb83. 00000400:00000010:0.0:1788844929.014219:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000a2016c52. 00000400:00000010:0.0:1788844929.014220:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000080a083d2. 00000400:00000010:0.0:1788844929.014224:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000001c0ab122. 00000400:00000010:0.0:1788844929.014225:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000b906f0cd. 00000400:00000010:0.0:1788844929.014227:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002c3ab64d. 00000400:00000010:0.0:1788844929.014235:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000001775e3f2. 00000400:00000010:0.0:1788844929.014239:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000cce5a98d. 00000400:00000010:0.0:1788844929.014241:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000019048ef3. 00000400:00000010:0.0:1788844929.014243:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000b6b7f8e5. 00000400:00000010:0.0:1788844929.014244:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000506fd682. 00000400:00000010:0.0:1788844929.014247:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000078e45bfe. 00000400:00000010:0.0:1788844929.014250:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000016de2d09. 00000400:00000010:0.0:1788844929.014252:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000003f304197. 00000400:00000010:0.0:1788844929.014253:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000003242f3c2. 00000400:00000010:0.0:1788844929.014257:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000073a778bd. 00000400:00000010:0.0:1788844929.014260:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000eb39fade. 00000400:00000010:0.0:1788844929.014261:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000085a8048d. 00000400:00000010:0.0:1788844929.014263:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000a4bf5b23. 00000400:00000010:0.0:1788844929.014265:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000008608d7f3. 00000400:00000010:0.0:1788844929.014267:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000000dffe2c6. 00000400:00000010:0.0:1788844929.014268:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000006a949bd. 00000400:00000010:0.0:1788844929.014270:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000084c131b0. 00000400:00000010:0.0:1788844929.014273:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000001c3d3225. 00000400:00000010:0.0:1788844929.014276:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000003262d3ee. 00000400:00000010:0.0:1788844929.014278:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000005904f260. 00000400:00000010:0.0:1788844929.014279:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000027ef9691. 00000400:00000010:0.0:1788844929.014282:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000048876783. 00000400:00000010:0.0:1788844929.014284:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000002c4fdf2f. 00000400:00000010:0.0:1788844929.014286:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000000a5e12c7. 00000400:00000010:0.0:1788844929.014300:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000992ba38a. 00000400:00000010:0.0:1788844929.014304:0:70068:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000beeb68e0. 00000400:00000010:0.0:1788844929.014307:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000001276e185. 00000400:00000010:0.0:1788844929.014308:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000004ce45170. 00000400:00000010:0.0:1788844929.014310:0:70068:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000072b9eea5. 00000400:00000010:0.0:1788844929.014312:0:70068:0:(lib-ptl.c:854:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals': 512 at 000000008d81bb62. 00000400:00000010:0.0:1788844929.014316:0:70068:0:(api-ni.c:1205:lnet_res_container_cleanup()) kfreed 'rec->rec_lh_hash': 65536 at 00000000ffacfe34. 00000400:00000010:0.0:1788844929.014331:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000b083c074. 00000400:00000010:0.0:1788844929.014338:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000109af81f. 00000400:00000010:0.0:1788844929.014340:0:70068:0:(api-ni.c:1205:lnet_res_container_cleanup()) kfreed 'rec->rec_lh_hash': 65536 at 00000000b244745d. 00000400:00000010:0.0:1788844929.014344:0:70068:0:(lib-msg.c:1308:lnet_msg_container_cleanup()) kfreed 'container->msc_finalizers': 32 at 0000000033014b15. 00000400:00000010:0.0:1788844929.014346:0:70068:0:(lib-msg.c:1314:lnet_msg_container_cleanup()) kfreed 'container->msc_resenders': 32 at 00000000c244c076. 00000400:00000010:0.0:1788844929.014348:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 00000000098e8f8c. 00000400:00000010:0.0:1788844929.014350:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000962292a7. 00000400:00000010:0.0:1788844929.014355:0:70068:0:(peer.c:85:lnet_peer_tables_destroy()) kfreed 'hash': 8192 at 000000004d481902. 00000400:00000010:0.0:1788844929.014364:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 00000000ffde6bcd. 00000400:00000010:0.0:1788844929.014366:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000024eb8ae2. 00000400:00000010:0.0:1788844929.014369:0:70068:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 00000000c33ab6cb. 00000400:00000010:0.0:1788844929.014370:0:70068:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000006e8e4304. 00000400:00000010:0.0:1788844929.014373:0:70068:0:(api-ni.c:700:lnet_destroy_remote_nets_table()) kfreed 'the_lnet.ln_remote_nets_hash': 1024 at 00000000f7f65837. 00000001:00000200:0.0:1788844929.014376:0:70068:0:(udsp.c:1126:lnet_udsp_destroy()) Destroying UDSPs in the system 00000001:02000400:1.0:1788844929.081407:0:70069:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure 00000400:00000010:3.0:1788844929.128897:0:70070:0:(api-ni.c:676:lnet_create_remote_nets_table()) alloc '(hash)': 1024 at 000000000a4cc18f. 00000400:00000010:3.0:1788844929.128908:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002210c718. 00000400:00000010:3.0:1788844929.128913:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 0000000027aa7e6b. 00000400:00000010:3.0:1788844929.128916:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000162133db. 00000400:00000010:3.0:1788844929.128918:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000162bf935. 00000400:00000010:3.0:1788844929.128923:0:70070:0:(peer.c:109:lnet_peer_tables_create()) alloc '(hash)': 8192 at 00000000aa6f72a4. 00000400:00000010:3.0:1788844929.128927:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005822650b. 00000400:00000010:3.0:1788844929.128929:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 000000007e5cd466. 00000400:00000010:3.0:1788844929.128932:0:70070:0:(lib-msg.c:1338:lnet_msg_container_setup()) alloc '(container->msc_finalizers)': 32 at 0000000093d98ddc. 00000400:00000010:3.0:1788844929.128934:0:70070:0:(lib-msg.c:1348:lnet_msg_container_setup()) alloc '(container->msc_resenders)': 32 at 00000000da79684b. 00000400:00000010:3.0:1788844929.129005:0:70070:0:(api-ni.c:1227:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 00000000343958f9. 00000400:00000010:3.0:1788844929.129024:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d85ba17b. 00000400:00000010:3.0:1788844929.129026:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ab0608cd. 00000400:00000010:3.0:1788844929.129606:0:70070:0:(api-ni.c:1227:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 000000000f3c9073. 00000400:00000010:3.0:1788844929.129632:0:70070:0:(lib-ptl.c:864:lnet_portals_create()) alloc '(the_lnet.ln_portals)': 512 at 0000000009eeb148. 00000400:00000010:3.0:1788844929.129643:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000746e11e9. 00000400:00000010:3.0:1788844929.129646:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000c04d4d2a. 00000400:00000010:3.0:1788844929.129648:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d89c206e. 00000400:00000010:3.0:1788844929.129653:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000beeb68e0. 00000400:00000010:3.0:1788844929.129656:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000d222071e. 00000400:00000010:3.0:1788844929.129658:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000044e89cc2. 00000400:00000010:3.0:1788844929.129660:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000099fb3615. 00000400:00000010:3.0:1788844929.129663:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000048876783. 00000400:00000010:3.0:1788844929.129665:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000018accbfb. 00000400:00000010:3.0:1788844929.129667:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001dbdd816. 00000400:00000010:3.0:1788844929.129668:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ff2f277f. 00000400:00000010:3.0:1788844929.129673:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000071fab5be. 00000400:00000010:3.0:1788844929.129675:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000008051c1b5. 00000400:00000010:3.0:1788844929.129676:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000c45e19d2. 00000400:00000010:3.0:1788844929.129677:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f230b7db. 00000400:00000010:3.0:1788844929.129683:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000008a53d52. 00000400:00000010:3.0:1788844929.129685:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000036ed3308. 00000400:00000010:3.0:1788844929.129687:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000b7ef994e. 00000400:00000010:3.0:1788844929.129689:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007cb86f97. 00000400:00000010:3.0:1788844929.129692:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000625a9453. 00000400:00000010:3.0:1788844929.129694:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000bac45014. 00000400:00000010:3.0:1788844929.129695:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000451a32e9. 00000400:00000010:3.0:1788844929.129697:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b0ad0c20. 00000400:00000010:3.0:1788844929.129700:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f8577e7d. 00000400:00000010:3.0:1788844929.129703:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000009aaf5135. 00000400:00000010:3.0:1788844929.129704:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000059567fb5. 00000400:00000010:3.0:1788844929.129706:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000007551669. 00000400:00000010:3.0:1788844929.129709:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001f03eb7b. 00000400:00000010:3.0:1788844929.129711:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000008b96fca4. 00000400:00000010:3.0:1788844929.129713:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001adc6de9. 00000400:00000010:3.0:1788844929.129714:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c55572a7. 00000400:00000010:3.0:1788844929.129719:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002bbfaf7b. 00000400:00000010:3.0:1788844929.129729:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000003a0abf39. 00000400:00000010:3.0:1788844929.129731:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000006157426e. 00000400:00000010:3.0:1788844929.129732:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003aae2451. 00000400:00000010:3.0:1788844929.129737:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f8b96da2. 00000400:00000010:3.0:1788844929.129739:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000826a768e. 00000400:00000010:3.0:1788844929.129741:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000004f388c65. 00000400:00000010:3.0:1788844929.129743:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ba5197df. 00000400:00000010:3.0:1788844929.129747:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000cce1b85e. 00000400:00000010:3.0:1788844929.129751:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a0b8df79. 00000400:00000010:3.0:1788844929.129752:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a60a864c. 00000400:00000010:3.0:1788844929.129754:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000056c2ca47. 00000400:00000010:3.0:1788844929.129758:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003abcc076. 00000400:00000010:3.0:1788844929.129760:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000d66762c1. 00000400:00000010:3.0:1788844929.129762:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000dde3929c. 00000400:00000010:3.0:1788844929.129763:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a0cabecd. 00000400:00000010:3.0:1788844929.129768:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000041fda1eb. 00000400:00000010:3.0:1788844929.129770:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000451395bc. 00000400:00000010:3.0:1788844929.129772:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001f3b2365. 00000400:00000010:3.0:1788844929.129774:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001b41631b. 00000400:00000010:3.0:1788844929.129777:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000091cba7af. 00000400:00000010:3.0:1788844929.130399:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000acaad1ed. 00000400:00000010:3.0:1788844929.130403:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e329392f. 00000400:00000010:3.0:1788844929.130406:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000aa726205. 00000400:00000010:3.0:1788844929.130410:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b075f539. 00000400:00000010:3.0:1788844929.130413:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ac1df15d. 00000400:00000010:3.0:1788844929.130415:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001681843c. 00000400:00000010:3.0:1788844929.130416:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d34169e7. 00000400:00000010:3.0:1788844929.130421:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ac6fdf8f. 00000400:00000010:3.0:1788844929.130423:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000703ea40e. 00000400:00000010:3.0:1788844929.130425:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000000140882c. 00000400:00000010:3.0:1788844929.130427:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000bbfdc362. 00000400:00000010:3.0:1788844929.130432:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000085d84fe0. 00000400:00000010:3.0:1788844929.130434:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000054413f2b. 00000400:00000010:3.0:1788844929.130443:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000072acab06. 00000400:00000010:3.0:1788844929.130445:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c93df21c. 00000400:00000010:3.0:1788844929.130450:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000666acba5. 00000400:00000010:3.0:1788844929.130452:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000d8807648. 00000400:00000010:3.0:1788844929.130454:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000003c313220. 00000400:00000010:3.0:1788844929.130456:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000022d0a19b. 00000400:00000010:3.0:1788844929.130460:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f2896859. 00000400:00000010:3.0:1788844929.130462:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000001c0ed6ef. 00000400:00000010:3.0:1788844929.130464:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000012f5b268. 00000400:00000010:3.0:1788844929.130466:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000047602523. 00000400:00000010:3.0:1788844929.130469:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001dc6f449. 00000400:00000010:3.0:1788844929.130471:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e2f13ec8. 00000400:00000010:3.0:1788844929.130473:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a8e07715. 00000400:00000010:3.0:1788844929.130475:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000043142ca4. 00000400:00000010:3.0:1788844929.130479:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ce632e44. 00000400:00000010:3.0:1788844929.130482:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000002cee4ceb. 00000400:00000010:3.0:1788844929.130483:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000091e2ad38. 00000400:00000010:3.0:1788844929.130485:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000644d0022. 00000400:00000010:3.0:1788844929.130488:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000365db3e4. 00000400:00000010:3.0:1788844929.130491:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000071282c6a. 00000400:00000010:3.0:1788844929.130492:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d24b9af7. 00000400:00000010:3.0:1788844929.130494:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000052f58b92. 00000400:00000010:3.0:1788844929.130497:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000026a45d92. 00000400:00000010:3.0:1788844929.130499:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000004e7cadcc. 00000400:00000010:3.0:1788844929.130501:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000493db505. 00000400:00000010:3.0:1788844929.130503:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b3a0e532. 00000400:00000010:3.0:1788844929.130507:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000dd0c6b59. 00000400:00000010:3.0:1788844929.130509:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000002df91953. 00000400:00000010:3.0:1788844929.130510:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000018346fcc. 00000400:00000010:3.0:1788844929.130512:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005894427f. 00000400:00000010:3.0:1788844929.130518:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000035c7b992. 00000400:00000010:3.0:1788844929.130520:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a9790428. 00000400:00000010:3.0:1788844929.130528:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ab70581a. 00000400:00000010:3.0:1788844929.130529:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009b3902d1. 00000400:00000010:3.0:1788844929.130534:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000c587f8d0. 00000400:00000010:3.0:1788844929.130536:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000006fcc5587. 00000400:00000010:3.0:1788844929.130538:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002226e38c. 00000400:00000010:3.0:1788844929.130539:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003c18bedc. 00000400:00000010:3.0:1788844929.130543:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004b3231fa. 00000400:00000010:3.0:1788844929.130545:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000c7bd6f1a. 00000400:00000010:3.0:1788844929.130547:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000044352509. 00000400:00000010:3.0:1788844929.130548:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003b563be1. 00000400:00000010:3.0:1788844929.130552:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000058d5f192. 00000400:00000010:3.0:1788844929.130554:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000024469acb. 00000400:00000010:3.0:1788844929.130555:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000072c72039. 00000400:00000010:3.0:1788844929.130557:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000037c36f6e. 00000400:00000010:3.0:1788844929.130562:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b7b45c03. 00000400:00000010:3.0:1788844929.130564:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000d90d0e03. 00000400:00000010:3.0:1788844929.130566:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005c6b28d3. 00000400:00000010:3.0:1788844929.130582:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000988b34c7. 00000400:00000010:3.0:1788844929.130587:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000012bb5fad. 00000400:00000010:3.0:1788844929.130589:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ac5f1b96. 00000400:00000010:3.0:1788844929.130590:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000000f30b4f1. 00000400:00000010:3.0:1788844929.130592:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000913d04c0. 00000400:00000010:3.0:1788844929.130596:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005aefee44. 00000400:00000010:3.0:1788844929.130599:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000072b9eea5. 00000400:00000010:3.0:1788844929.130601:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000046662d02. 00000400:00000010:3.0:1788844929.130602:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008e332516. 00000400:00000010:3.0:1788844929.130616:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004d481902. 00000400:00000010:3.0:1788844929.130621:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000992ba38a. 00000400:00000010:3.0:1788844929.130622:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000014991cc7. 00000400:00000010:3.0:1788844929.130624:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000cb18660e. 00000400:00000010:3.0:1788844929.130628:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005636bcd6. 00000400:00000010:3.0:1788844929.130630:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000027ef9691. 00000400:00000010:3.0:1788844929.130639:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002f9882bf. 00000400:00000010:3.0:1788844929.130641:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001276e185. 00000400:00000010:3.0:1788844929.130645:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000019bdd1d9. 00000400:00000010:3.0:1788844929.130647:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000084c131b0. 00000400:00000010:3.0:1788844929.130648:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000007060eeb1. 00000400:00000010:3.0:1788844929.130649:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002c4fdf2f. 00000400:00000010:3.0:1788844929.130653:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000595b4ee4. 00000400:00000010:3.0:1788844929.130655:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a4bf5b23. 00000400:00000010:3.0:1788844929.130656:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000004b77f11c. 00000400:00000010:3.0:1788844929.130658:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003262d3ee. 00000400:00000010:3.0:1788844929.130667:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e29cd06f. 00000400:00000010:3.0:1788844929.130670:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000003242f3c2. 00000400:00000010:3.0:1788844929.130672:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002551acd8. 00000400:00000010:3.0:1788844929.130673:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000dffe2c6. 00000400:00000010:3.0:1788844929.130676:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e492d794. 00000400:00000010:3.0:1788844929.130678:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000506fd682. 00000400:00000010:3.0:1788844929.130679:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000f355510d. 00000400:00000010:3.0:1788844929.130681:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000eb39fade. 00000400:00000010:3.0:1788844929.130684:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ea9620cd. 00000400:00000010:3.0:1788844929.130686:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000001775e3f2. 00000400:00000010:3.0:1788844929.130688:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000fb2d6842. 00000400:00000010:3.0:1788844929.130689:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000016de2d09. 00000400:00000010:3.0:1788844929.130693:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f4841a6c. 00000400:00000010:3.0:1788844929.130695:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000080a083d2. 00000400:00000010:3.0:1788844929.130696:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000b308ad10. 00000400:00000010:3.0:1788844929.130698:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000019048ef3. 00000400:00000010:3.0:1788844929.130707:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002ca18eb7. 00000400:00000010:3.0:1788844929.130709:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000005ce728f6. 00000400:00000010:3.0:1788844929.130711:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000009116a73c. 00000400:00000010:3.0:1788844929.130713:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b906f0cd. 00000400:00000010:3.0:1788844929.130717:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000003ba3b00. 00000400:00000010:3.0:1788844929.130720:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000096649c0f. 00000400:00000010:3.0:1788844929.130725:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000cf697eee. 00000400:00000010:3.0:1788844929.130726:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000614fdb83. 00000400:00000010:3.0:1788844929.130730:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000062b2d79e. 00000400:00000010:3.0:1788844929.130733:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000057beef80. 00000400:00000010:3.0:1788844929.130734:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ee8dfa2e. 00000400:00000010:3.0:1788844929.130736:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000803935c1. 00000400:00000010:3.0:1788844929.130739:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004950b76d. 00000400:00000010:3.0:1788844929.130741:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a9fa0434. 00000400:00000010:3.0:1788844929.130743:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002577d682. 00000400:00000010:3.0:1788844929.130745:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002d892f93. 00000400:00000010:3.0:1788844929.130754:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d8f2cce8. 00000400:00000010:3.0:1788844929.130757:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e2d0109f. 00000400:00000010:3.0:1788844929.130759:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000161d5ed6. 00000400:00000010:3.0:1788844929.130761:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000b0cf052. 00000400:00000010:3.0:1788844929.130765:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e5717462. 00000400:00000010:3.0:1788844929.130767:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000004f5b511f. 00000400:00000010:3.0:1788844929.130768:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000741f4313. 00000400:00000010:3.0:1788844929.130770:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007b0c6120. 00000400:00000010:3.0:1788844929.130774:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f0f3900e. 00000400:00000010:3.0:1788844929.130776:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000040fecfea. 00000400:00000010:3.0:1788844929.130778:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000fcbb744a. 00000400:00000010:3.0:1788844929.130800:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ed069f39. 00000400:00000010:3.0:1788844929.130805:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001ba26530. 00000400:00000010:3.0:1788844929.130807:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000006bdf38eb. 00000400:00000010:3.0:1788844929.130809:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000098462bc. 00000400:00000010:3.0:1788844929.130811:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007f2c1e88. 00000400:00000010:3.0:1788844929.130821:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d7072bc3. 00000400:00000010:3.0:1788844929.130824:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000004ee91bb2. 00000400:00000010:3.0:1788844929.130825:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000feebb32e. 00000400:00000010:3.0:1788844929.130827:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fe307cec. 00000400:00000010:3.0:1788844929.130830:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b555330b. 00000400:00000010:3.0:1788844929.130832:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000c2dbc3d2. 00000400:00000010:3.0:1788844929.130837:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000363a54ae. 00000400:00000010:3.0:1788844929.130839:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009a7ce84e. 00000400:00000010:3.0:1788844929.130843:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000dc206ed9. 00000400:00000010:3.0:1788844929.130846:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000110d9712. 00000400:00000010:3.0:1788844929.130847:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000c2f823b2. 00000400:00000010:3.0:1788844929.130848:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006c80d182. 00000400:00000010:3.0:1788844929.130852:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d584d457. 00000400:00000010:3.0:1788844929.130855:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000b3a01587. 00000400:00000010:3.0:1788844929.130856:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a9db2fb4. 00000400:00000010:3.0:1788844929.130858:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000508a48de. 00000400:00000010:3.0:1788844929.130866:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003c7946d5. 00000400:00000010:3.0:1788844929.130871:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000071d2f252. 00000400:00000010:3.0:1788844929.130872:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ba407268. 00000400:00000010:3.0:1788844929.130874:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000004b1d41d. 00000400:00000010:3.0:1788844929.130877:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000049f837f7. 00000400:00000010:3.0:1788844929.130879:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000008f567d23. 00000400:00000010:3.0:1788844929.130881:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e964c8d9. 00000400:00000010:3.0:1788844929.130882:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ad5db28e. 00000400:00000010:3.0:1788844929.130885:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004c570512. 00000400:00000010:3.0:1788844929.130887:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000004f3e8e36. 00000400:00000010:3.0:1788844929.130889:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000023af65e. 00000400:00000010:3.0:1788844929.130891:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004b754388. 00000400:00000010:3.0:1788844929.130894:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006051c759. 00000400:00000010:3.0:1788844929.130896:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000000397cbde. 00000400:00000010:3.0:1788844929.130897:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000baf33e5d. 00000400:00000010:3.0:1788844929.130900:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ad411170. 00000400:00000010:3.0:1788844929.130907:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e38571ac. 00000400:00000010:3.0:1788844929.130911:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000aa243cd0. 00000400:00000010:3.0:1788844929.130912:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000013d74354. 00000400:00000010:3.0:1788844929.130914:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000099e3021d. 00000400:00000010:3.0:1788844929.130918:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b01d5179. 00000400:00000010:3.0:1788844929.130920:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ac19a753. 00000400:00000010:3.0:1788844929.130925:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005a1afdca. 00000400:00000010:3.0:1788844929.130926:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e9fd9aec. 00000400:00000010:3.0:1788844929.130930:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a57cc282. 00000400:00000010:3.0:1788844929.130932:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000c0c8690a. 00000400:00000010:3.0:1788844929.130934:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000025c09be2. 00000400:00000010:3.0:1788844929.130935:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000057d5d3e5. 00000400:00000010:3.0:1788844929.130938:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000042aaebca. 00000400:00000010:3.0:1788844929.130940:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000052efff7. 00000400:00000010:3.0:1788844929.130942:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000028083766. 00000400:00000010:3.0:1788844929.130944:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000048798ae8. 00000400:00000010:3.0:1788844929.130953:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000052a76679. 00000400:00000010:3.0:1788844929.130956:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000013cd7efe. 00000400:00000010:3.0:1788844929.130958:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000004f96d58c. 00000400:00000010:3.0:1788844929.130959:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002601722b. 00000400:00000010:3.0:1788844929.130963:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002383e933. 00000400:00000010:3.0:1788844929.130965:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000049eef0e4. 00000400:00000010:3.0:1788844929.130966:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000eed24453. 00000400:00000010:3.0:1788844929.130968:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000166fd25d. 00000400:00000010:3.0:1788844929.130971:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004c3f562e. 00000400:00000010:3.0:1788844929.130974:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000004a2fa563. 00000400:00000010:3.0:1788844929.130975:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a89d7f34. 00000400:00000010:3.0:1788844929.130977:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b433b6a4. 00000400:00000010:3.0:1788844929.130981:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f8fd273d. 00000400:00000010:3.0:1788844929.130984:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000c23603ce. 00000400:00000010:3.0:1788844929.130986:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d5e902c4. 00000400:00000010:3.0:1788844929.130988:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000085da7055. 00000400:00000010:3.0:1788844929.130997:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000058c19110. 00000400:00000010:3.0:1788844929.131001:0:70070:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000006e12e825. 00000400:00000010:3.0:1788844929.131003:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000006591c073. 00000400:00000010:3.0:1788844929.131005:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000019c4c10e. 00000400:00000010:3.0:1788844929.131009:0:70070:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000070bb64a1. 00000400:00000010:3.0:1788844929.131012:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005f6e227b. 00000400:00000010:3.0:1788844929.131017:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005659adaf. 00000400:00000010:3.0:1788844929.131021:0:70070:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 00000000c1d7e28a. 00000400:00000010:3.0:1788844929.131025:0:70070:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 560 at 00000000cb247215. 00000400:00000010:3.0:1788844929.131028:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000009ccf7bec. 00000400:00000010:3.0:1788844929.131029:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d0597093. 00000400:00000010:3.0:1788844929.131031:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000bb786270. 00000400:00000010:3.0:1788844929.131032:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003a151efd. 00000400:00000010:3.0:1788844929.131049:0:70070:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 41 at 00000000113453fe. 00000400:00000010:3.0:1788844929.131054:0:70070:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000009e812ea7. 00000400:00000010:3.0:1788844929.131069:0:70070:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000312d23da. 00000400:00000200:3.0:1788844929.131094:0:70070:0:(lib-ptl.c:918:LNetSetLazyPortal()) Setting portal 0 lazy 00000400:00000010:3.0:1788844929.131099:0:70070:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000023f07e03. 00000400:00000010:3.0:1788844929.131102:0:70070:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000078ad9a61. 00000400:00000010:3.0:1788844929.131105:0:70070:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000090a3025. 00000400:00000200:3.0:1788844929.131107:0:70070:0:(api-ni.c:2383:lnet_push_target_post()) posted push target 0000000023f07e03 00000400:00000200:3.0:1788844929.131109:0:70070:0:(api-ni.c:2340:lnet_push_target_resize()) nbytes 272 success 00000400:00000010:3.0:1788844929.131112:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000dbcf86c9. 00000400:00000010:3.0:1788844929.131114:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007f53d27e. 00000400:00000010:3.0:1788844929.131117:0:70070:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002780725a. 00000400:00000010:3.0:1788844929.131119:0:70070:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000571bf052. 00000400:00000200:3.0:1788844929.133433:0:70070:0:(peer.c:4457:lnet_peer_discovery_start()) discovery start: 0 00000400:00000200:0.0:1788844929.133763:0:70072:0:(peer.c:4283:lnet_peer_discovery()) started 00000400:00000010:2.0:1788844929.150730:0:70073:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 0000000041fbcbc5. 00000400:00000010:2.0:1788844929.150754:0:70073:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 0000000041fbcbc5. 00000400:00000010:2.0:1788844929.151736:0:70073:0:(api-ni.c:7409:lnet_route_show_start()) alloc '(rlist)': 16 at 00000000db156cca. 00000400:00000010:2.0:1788844929.151755:0:70073:0:(api-ni.c:7309:lnet_route_show_done()) kfreed 'rlist': 16 at 00000000db156cca. 00000400:00000010:2.0:1788844929.152398:0:70073:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 236 at 0000000085e5a865. 00000400:00000010:2.0:1788844929.152407:0:70073:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 236 at 0000000085e5a865. 00000400:00000010:2.0:1788844929.152754:0:70073:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 00000000db156cca. 00000400:00000010:2.0:1788844929.152768:0:70073:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 00000000db156cca. 00000400:00000010:2.0:1788844929.153479:0:70073:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 12 at 00000000db156cca. 00000400:00000010:2.0:1788844929.153488:0:70073:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 12 at 00000000db156cca. 00000400:00000010:2.0:1788844929.155738:0:70073:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 32 at 000000005ea16edd. 00000001:00000200:2.0:1788844929.155747:0:70073:0:(udsp.c:863:lnet_udsp_get_policy()) Get UDSP at idx = 0 00000001:00000200:2.0:1788844929.155749:0:70073:0:(udsp.c:878:lnet_udsp_get_policy()) Found UDSP (000000007d9b531d) 00000400:00000010:2.0:1788844929.155752:0:70073:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 32 at 000000005ea16edd. 00000001:02000400:2.0:1788844929.177051:0:70074:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp --if ens2 00000400:00000010:0.0:1788844929.218924:0:70075:0:(api-ni.c:6877:lnet_net_cmd()) alloc '(conf)': 808 at 00000000527854b3. 00000400:00000010:0.0:1788844929.218944:0:70075:0:(api-ni.c:6555:lnet_genl_parse_local_ni()) alloc '(tun)': 172 at 00000000cdc34feb. 00000400:00000010:0.0:1788844929.218954:0:70075:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 00000000e37e2096. 00000400:00000010:0.0:1788844929.218959:0:70075:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 560 at 00000000ece4e79b. 00000400:00000010:0.0:1788844929.218963:0:70075:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000007ef6aab4. 00000400:00000010:0.0:1788844929.218969:0:70075:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000071598e2c. 00000400:00000010:0.0:1788844929.218971:0:70075:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005b3529e4. 00000400:00000010:0.0:1788844929.218973:0:70075:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000006fd6af5. 00000400:00000010:0.0:1788844929.218977:0:70075:0:(config.c:397:lnet_ni_add_interface()) alloc '(ni->ni_interface)': 5 at 00000000c8a932d6. 00000400:00000010:0.0:1788844929.218988:0:70075:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000000743d23c. 00000400:00000010:0.0:1788844929.218991:0:70075:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 000000002781177b. 00000800:00000200:2.0:1788844929.220547:0:70075:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=lo ifindex 1 state 0 ns 000000004dc0f701 00000800:00000200:2.0:1788844929.220556:0:70075:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=lo ifindex 1 state 0 ns 000000004dc0f701 00000800:00000200:2.0:1788844929.220564:0:70075:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=ens2 ifindex 2 state 6 ns 000000004dc0f701 00000800:00000200:2.0:1788844929.220567:0:70075:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=ens2 ifindex 2 state 6 ns 000000004dc0f701 00000800:00000010:2.0:1788844929.220682:0:70075:0:(socklnd.c:2621:ksocknal_startup()) alloc '(net)': 208 at 0000000085e5a865. 00000800:00000200:2.0:1788844929.220696:0:70075:0:(socklnd_modparams.c:295:ksocklnd_lookup_conns_per_peer()) intf ens2 speed -1 00000400:00000200:2.0:1788844929.221700:0:70075:0:(config.c:1657:lnet_get_link_status_locked()) device idx 1 get_link 2 00000400:00000200:2.0:1788844929.221762:0:70075:0:(acceptor.c:209:lnet_acceptor_add_socket()) Added socket on ens2:988 (IP: 192.168.202.29) 00000400:02000000:2.0:1788844929.221777:0:70075:0:(api-ni.c:2823:lnet_startup_lndni()) Added LNI 192.168.202.29@tcp [8/256/0/180] 00000400:00000010:2.0:1788844929.223994:0:70075:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 57 at 00000000c6ebeb39. 00000400:00000010:2.0:1788844929.224006:0:70075:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000009dd382c4. 00000400:00000010:2.0:1788844929.224033:0:70075:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000016f16ec8. 00000400:00000010:2.0:1788844929.225196:0:70075:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000009e812ea7. 00000400:00000200:2.0:1788844929.225203:0:70075:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000312d23da 00000400:00000010:2.0:1788844929.225205:0:70075:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000312d23da. 00000400:00000010:2.0:1788844929.225210:0:70075:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 41 at 00000000113453fe. 00000400:00000010:2.0:1788844929.225300:0:70075:0:(api-ni.c:6852:lnet_genl_parse_local_ni()) kfreed 'tun': 172 at 00000000cdc34feb. 00000400:00000010:2.0:1788844929.225306:0:70075:0:(api-ni.c:6994:lnet_net_cmd()) kfreed 'conf': 808 at 00000000527854b3. 00000400:00000010:3.0:1788844929.390146:0:70089:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000082c3b6a. 00000400:00000010:3.0:1788844929.390234:0:70089:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000082c3b6a. 00000001:02000400:3.0:1788844930.464206:0:70195:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp1 --if ens2 00000400:00000010:2.0:1788844930.481307:0:70196:0:(api-ni.c:6877:lnet_net_cmd()) alloc '(conf)': 808 at 00000000a2fbf96b. 00000400:00000010:2.0:1788844930.481328:0:70196:0:(api-ni.c:6555:lnet_genl_parse_local_ni()) alloc '(tun)': 172 at 00000000d2c0e401. 00000400:00000010:2.0:1788844930.481338:0:70196:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 00000000315b20ae. 00000400:00000010:2.0:1788844930.481342:0:70196:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 560 at 0000000031d7cbc7. 00000400:00000010:2.0:1788844930.481346:0:70196:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000008b9f3c8f. 00000400:00000010:2.0:1788844930.481351:0:70196:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008d6cb046. 00000400:00000010:2.0:1788844930.481353:0:70196:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000fb1612fc. 00000400:00000010:2.0:1788844930.481355:0:70196:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005f464acc. 00000400:00000010:2.0:1788844930.481359:0:70196:0:(config.c:397:lnet_ni_add_interface()) alloc '(ni->ni_interface)': 5 at 0000000041fbcbc5. 00000800:00000010:2.0:1788844930.481368:0:70196:0:(socklnd.c:2621:ksocknal_startup()) alloc '(net)': 208 at 00000000b8270d8a. 00000800:00000200:2.0:1788844930.481380:0:70196:0:(socklnd_modparams.c:295:ksocklnd_lookup_conns_per_peer()) intf ens2 speed -1 00000400:00000200:2.0:1788844930.481386:0:70196:0:(config.c:1657:lnet_get_link_status_locked()) device idx 1 get_link 2 00000400:00000200:2.0:1788844930.481433:0:70196:0:(acceptor.c:209:lnet_acceptor_add_socket()) Added socket on ens2:988 (IP: 192.168.202.29) 00000400:02000000:2.0:1788844930.481610:0:70196:0:(api-ni.c:2823:lnet_startup_lndni()) Added LNI 192.168.202.29@tcp1 [8/256/0/180] 00000400:00000010:3.0:1788844930.490870:0:70196:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 73 at 0000000032f97331. 00000400:00000010:3.0:1788844930.490881:0:70196:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000034148b21. 00000400:00000010:3.0:1788844930.490886:0:70196:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cba564a7. 00000400:00000010:3.0:1788844930.490899:0:70196:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000009dd382c4. 00000400:00000200:3.0:1788844930.490901:0:70196:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000016f16ec8 00000400:00000010:3.0:1788844930.490903:0:70196:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 0000000016f16ec8. 00000400:00000010:3.0:1788844930.490906:0:70196:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 57 at 00000000c6ebeb39. 00000400:00000010:3.0:1788844930.491015:0:70196:0:(api-ni.c:6852:lnet_genl_parse_local_ni()) kfreed 'tun': 172 at 00000000d2c0e401. 00000400:00000010:3.0:1788844930.491022:0:70196:0:(api-ni.c:6994:lnet_net_cmd()) kfreed 'conf': 808 at 00000000a2fbf96b. 00000400:00000010:0.0:1788844930.533471:0:70201:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 0000000018ffce37. 00000400:00000010:0.0:1788844930.533539:0:70201:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 0000000018ffce37. 00000400:00000010:3.0:1788844931.656717:0:70319:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000082c3b6a. 00000400:00000010:3.0:1788844931.656847:0:70319:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000082c3b6a. 00000001:02000400:3.0:1788844931.681626:0:70321:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl discover 192.168.202.29@tcp 00000400:00000010:2.0:1788844931.709183:0:70322:0:(peer.c:230:lnet_peer_alloc()) alloc '(lp)': 360 at 00000000d558236e. 00000400:00000200:2.0:1788844931.709197:0:70322:0:(peer.c:266:lnet_peer_alloc()) 00000000d558236e nid 192.168.202.29@tcp 00000400:00000010:2.0:1788844931.709201:0:70322:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 00000000d55bb4a0. 00000400:00000200:2.0:1788844931.709210:0:70322:0:(peer.c:203:lnet_peer_net_alloc()) 00000000d55bb4a0 net tcp 00000400:00000010:2.0:1788844931.709214:0:70322:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 00000000fa760abd. 00000400:00000200:2.0:1788844931.709218:0:70322:0:(peer.c:184:lnet_peer_ni_alloc()) 00000000fa760abd nid 192.168.202.29@tcp 00000400:00000200:2.0:1788844931.709225:0:70322:0:(peer.c:1804:lnet_peer_attach_peer_ni()) peer 192.168.202.29@tcp NID 192.168.202.29@tcp flags 0x0 00000400:00000200:2.0:1788844931.709229:0:70322:0:(peer.c:2183:lnet_peer_ni_traffic_add()) peer 192.168.202.29@tcp: 0 00000400:00000200:2.0:1788844931.709262:0:70322:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.202.29@tcp: 0 00000400:00000200:2.0:1788844931.709275:0:70322:0:(peer.c:2935:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1788844931.709510:0:70072:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1788844931.709517:0:70072:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.202.29@tcp(00000000d558236e) state 0x40 00000400:00000010:1.0:1788844931.709523:0:70072:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000a0d9d2a7. 00000400:00000010:1.0:1788844931.709553:0:70072:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000097db671e. 00000400:00000010:1.0:1788844931.709573:0:70072:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000c5c9fad8. 00000400:00000200:1.0:1788844931.709577:0:70072:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.29@tcp 00000400:00000200:1.0:1788844931.709588:0:70072:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.29@tcp local destination 00000400:00000200:1.0:1788844931.709594:0:70072:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.29@tcp 00000400:00000200:1.0:1788844931.709596:0:70072:0:(lib-move.c:1874:lnet_set_non_mr_pref_nid()) Setting preferred local NID 192.168.202.29@tcp on NMR peer 192.168.202.29@tcp 00000400:00000200:1.0:1788844931.709599:0:70072:0:(peer.c:1171:lnet_peer_ni_set_non_mr_pref_nid()) peer 192.168.202.29@tcp nid 192.168.202.29@tcp: 0 00000400:00000200:1.0:1788844931.709603:0:70072:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [32:32:256:4294967295] 192.168.202.29@tcp LPNI seq info [1:1:8:4294967295] 00000400:00000200:1.0:1788844931.709609:0:70072:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:1.0:1788844931.709616:0:70072:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : GET try# 0 00000800:00000200:1.0:1788844931.709622:0:70072:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.29@tcp 00000800:00000010:1.0:1788844931.709626:0:70072:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000085b28da0. 00000800:00000010:1.0:1788844931.709635:0:70072:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 0000000001785d6c. 00000800:00000010:1.0:1788844931.709638:0:70072:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 00000000ab14a5ab. 00000800:00000200:1.0:1788844931.709643:0:70072:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001785d6c] -> 12345-192.168.202.29@tcp (2) 00000800:00000200:1.1:1788844931.709647:0:70072:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001785d6c] -> 12345-192.168.202.29@tcp (2) 00000400:00000200:1.0:1788844931.709668:0:70072:0:(peer.c:4022:lnet_peer_send_ping()) peer 192.168.202.29@tcp 00000400:00000200:1.0:1788844931.709670:0:70072:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.202.29@tcp(00000000d558236e) state 0x240 rc 0 00000800:00000010:2.0:1788844931.719351:0:70076:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 000000006ade1324. 00000800:00000010:2.0:1788844931.719362:0:70076:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 00000000c011048d. 00000800:00000010:2.0:1788844931.719369:0:70076:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000ebd2e1d2. 00000400:00000200:0.0:1788844931.719435:0:70084:0:(acceptor.c:578:lnet_acceptor()) Accepted connection on ens2:988 00000400:00000200:1.0:1788844931.719524:0:70084:0:(acceptor.c:502:lnet_accept()) Accept 192.168.202.29@tcp from 255.3.0.2 00000800:00000010:1.0:1788844931.719528:0:70084:0:(socklnd.c:737:ksocknal_accept()) alloc '(cr)': 32 at 00000000c98474c7. 00000800:00000010:3.0:1788844931.719667:0:70077:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 00000000867cf3f8. 00000800:00000010:3.0:1788844931.719672:0:70077:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 000000007cd424fd. 00000800:00000010:2.0:1788844931.719761:0:70076:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000ebd2e1d2. 00000800:00000010:2.0:1788844931.720271:0:70077:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 000000007bcabe55. 00000800:00000200:2.1:1788844931.720280:0:70077:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001785d6c] -> 12345-192.168.202.29@tcp (2) 00000800:00000200:2.1:1788844931.720284:0:70077:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.202.29@tcp 000000007bcabe55 deleted 00000800:00000010:2.1:1788844931.720286:0:70077:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 000000007bcabe55. 00000800:00000200:2.1:1788844931.720291:0:70077:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:2.1:1788844931.720297:0:70077:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000200:2.1:1788844931.720302:0:70077:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000085b28da0 type 2, nob 96 niov 1 nkiov 0 00000800:00000200:2.0:1788844931.720306:0:70077:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.202.29@tcp p 3.x 192.168.202.29 -> 192.168.202.29:1023 incarnation:1788844929220685004 sched[0] 00000800:00000010:2.0:1788844931.720311:0:70077:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000ebd2e1d2. 00000800:00000010:2.0:1788844931.720412:0:70077:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000ebd2e1d2. 00000800:00000010:2.0:1788844931.720416:0:70077:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 000000007cd424fd. 00000800:00000010:2.0:1788844931.720450:0:70077:0:(socklnd_cb.c:2179:ksocknal_connd()) kfreed 'cr': 32 at 00000000c98474c7. 00000800:00000200:1.1:1788844931.720642:0:70076:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:1.0:1788844931.720649:0:70076:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.202.29@tcp p 3.x 192.168.202.29 -> 192.168.202.29:988 incarnation:1788844929220685004 sched[0] 00000800:00000010:1.0:1788844931.720658:0:70076:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 00000000c011048d. 00000400:00000200:1.0:1788844931.720729:0:70076:0:(lib-socket.c:438:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.202.29:988 00000400:00000200:1.0:1788844931.720938:0:70084:0:(acceptor.c:578:lnet_acceptor()) Accepted connection on ens2:988 00000400:00000200:1.0:1788844931.720997:0:70084:0:(acceptor.c:502:lnet_accept()) Accept 192.168.202.29@tcp from 254.3.0.2 00000800:00000010:1.0:1788844931.721000:0:70084:0:(socklnd.c:737:ksocknal_accept()) alloc '(cr)': 32 at 00000000b642eb51. 00000800:00000010:1.0:1788844931.721029:0:70079:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 0000000044dd9a9a. 00000800:00000010:1.0:1788844931.721032:0:70079:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 00000000931fb5c9. 00000800:00000010:1.0:1788844931.721043:0:70076:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 00000000e3ab8c90. 00000800:00000010:1.0:1788844931.721046:0:70076:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 0000000090272e43. 00000800:00000010:1.0:1788844931.721049:0:70076:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000ef77e56a. 00000800:00000010:1.0:1788844931.721092:0:70079:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 000000005b39715a. 00000800:00000200:1.1:1788844931.721097:0:70079:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001785d6c] -> 12345-192.168.202.29@tcp (4) 00000800:00000200:1.1:1788844931.721100:0:70079:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.202.29@tcp 000000005b39715a deleted 00000800:00000010:1.1:1788844931.721102:0:70079:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 000000005b39715a. 00000800:00000200:1.1:1788844931.721104:0:70079:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 3, ksnr_connected a ksnr_max_conns 1 00000800:00000200:1.0:1788844931.721108:0:70079:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.202.29@tcp p 3.x 192.168.202.29 -> 192.168.202.29:1022 incarnation:1788844929220685004 sched[0] 00000800:00000010:1.0:1788844931.721112:0:70079:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000dd57f84e. 00000800:00000010:1.0:1788844931.721139:0:70079:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000dd57f84e. 00000800:00000010:1.0:1788844931.721143:0:70079:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 00000000931fb5c9. 00000800:00000010:1.0:1788844931.721154:0:70079:0:(socklnd_cb.c:2179:ksocknal_connd()) kfreed 'cr': 32 at 00000000b642eb51. 00000800:00000010:1.0:1788844931.721192:0:70076:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000ef77e56a. 00000800:00000200:1.1:1788844931.721201:0:70076:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 2, ksnr_connected e ksnr_max_conns 1 00000800:00000200:1.0:1788844931.721205:0:70076:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.202.29@tcp p 3.x 192.168.202.29 -> 192.168.202.29:988 incarnation:1788844929220685004 sched[0] 00000800:00000010:1.0:1788844931.721208:0:70076:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 0000000090272e43. 00000800:00000200:3.0:1788844931.721750:0:70082:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788844931.721755:0:70082:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000085b28da0. 00000400:00000200:3.0:1788844931.721759:0:70082:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788844931.721764:0:70082:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: GET: OK 00000400:00000200:3.0:1788844931.721770:0:70082:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1788844931.721773:0:70082:0:(peer.c:3241:lnet_discovery_event_send()) Ping Send to 192.168.202.29@tcp: 0 00000400:00000200:3.0:1788844931.722179:0:70082:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : GET - for me 00000400:00000200:3.0:1788844931.722210:0:70082:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.29@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1788844931.722228:0:70082:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.29@tcp of length 64/272 into md 0xd [1] + 0 00000400:00000200:3.0:1788844931.722240:0:70082:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.29@tcp to NMR: 192.168.202.29@tcp local destination 00000400:00000200:3.0:1788844931.722245:0:70082:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [33:33:256:4294967295] 192.168.202.29@tcp LPNI seq info [2:2:8:4294967295] 00000400:00000200:3.0:1788844931.722254:0:70082:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) 192.168.202.29@tcp : REPLY try# 0 00000800:00000200:3.0:1788844931.722259:0:70082:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.202.29@tcp 00000800:00000010:3.0:1788844931.722273:0:70082:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000d9d7246b. 00000800:00000200:3.0:1788844931.722279:0:70082:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001785d6c] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:3.0:1788844931.722285:0:70082:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000200:3.0:1788844931.722288:0:70082:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000d9d7246b type 3, nob 160 niov 1 nkiov 1 00000800:00000200:3.0:1788844931.722333:0:70082:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788844931.722335:0:70082:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000d9d7246b. 00000400:00000200:3.0:1788844931.722338:0:70082:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788844931.722342:0:70082:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: REPLY: OK 00000400:00000200:3.0:1788844931.722357:0:70082:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : REPLY - for me 00000400:00000200:3.0:1788844931.722365:0:70082:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.29@tcp: Reply from 12345-192.168.202.29@tcp of length 64/64 into md 0x11 00000400:00000200:3.0:1788844931.722371:0:70082:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788844931.722374:0:70082:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: REPLY: OK 00000400:00000200:3.0:1788844931.722384:0:70082:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000097db671e 00000400:00000010:3.0:1788844931.722387:0:70082:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 0000000097db671e. 00000400:00000200:3.0:1788844931.722390:0:70082:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 3 00000400:00000200:3.0:1788844931.722393:0:70082:0:(peer.c:3119:lnet_discovery_event_reply()) Peer 192.168.202.29@tcp has discovery enabled 00000400:00000200:3.0:1788844931.722396:0:70082:0:(peer.c:3145:lnet_discovery_event_reply()) peer 192.168.202.29@tcp(00000000d558236e) is MR capable 00000400:00000200:3.0:1788844931.722399:0:70082:0:(peer.c:1196:lnet_peer_ni_clr_non_mr_pref_nid()) peer 192.168.202.29@tcp: 0 00000400:00000200:3.0:1788844931.722402:0:70082:0:(peer.c:3200:lnet_discovery_event_reply()) peer 192.168.202.29@tcp data present 12. state = 0x241 00000400:00000200:3.0:1788844931.722406:0:70082:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.202.29@tcp: -114 00000400:00000200:2.0:1788844931.725420:0:70072:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1788844931.725435:0:70072:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.202.29@tcp(00000000d558236e) state 0xc1 00000400:00000010:2.0:1788844931.725452:0:70072:0:(peer.c:3474:lnet_peer_merge_data()) alloc '(curnis)': 60 at 00000000ebd2e1d2. 00000400:00000010:2.0:1788844931.725457:0:70072:0:(peer.c:3475:lnet_peer_merge_data()) alloc '(addnis)': 72 at 00000000d3a31ee7. 00000400:00000010:2.0:1788844931.725459:0:70072:0:(peer.c:3476:lnet_peer_merge_data()) alloc '(delnis)': 60 at 00000000d5dded7f. 00000400:00000010:2.0:1788844931.725471:0:70072:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 00000000b03adc18. 00000400:00000200:2.0:1788844931.725478:0:70072:0:(peer.c:184:lnet_peer_ni_alloc()) 00000000b03adc18 nid 192.168.202.29@tcp1 00000400:00000010:2.0:1788844931.725481:0:70072:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 0000000014b7cb64. 00000400:00000200:2.0:1788844931.725484:0:70072:0:(peer.c:203:lnet_peer_net_alloc()) 0000000014b7cb64 net tcp1 00000400:00000200:2.0:1788844931.725490:0:70072:0:(peer.c:1804:lnet_peer_attach_peer_ni()) peer 192.168.202.29@tcp NID 192.168.202.29@tcp1 flags 0x11 00000400:00000010:2.0:1788844931.725494:0:70072:0:(peer.c:3623:lnet_peer_merge_data()) kfreed 'curnis': 60 at 00000000ebd2e1d2. 00000400:00000010:2.0:1788844931.725499:0:70072:0:(peer.c:3624:lnet_peer_merge_data()) kfreed 'addnis': 72 at 00000000d3a31ee7. 00000400:00000010:2.0:1788844931.725501:0:70072:0:(peer.c:3625:lnet_peer_merge_data()) kfreed 'delnis': 60 at 00000000d5dded7f. 00000400:00000010:2.0:1788844931.725505:0:70072:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000a0d9d2a7. 00000400:00000200:2.0:1788844931.725509:0:70072:0:(peer.c:3628:lnet_peer_merge_data()) peer 192.168.202.29@tcp (00000000d558236e): 0 00000400:00000200:2.0:1788844931.725512:0:70072:0:(peer.c:3940:lnet_peer_data_present()) peer 192.168.202.29@tcp(00000000d558236e): 0. state = 0x141 00000400:00000200:2.0:1788844931.725534:0:70072:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.202.29@tcp(00000000d558236e) state 0x141 rc 1 00000400:00000200:2.0:1788844931.725537:0:70072:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.202.29@tcp(00000000d558236e) state 0x141 00000400:00000010:2.0:1788844931.725555:0:70072:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001ebb8e73. 00000400:00000010:2.0:1788844931.725583:0:70072:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000fb1a1c64. 00000400:00000200:2.0:1788844931.725589:0:70072:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.29@tcp 00000400:00000200:2.0:1788844931.725604:0:70072:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.29@tcp to MR: 192.168.202.29@tcp local destination 00000400:00000200:2.0:1788844931.725611:0:70072:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [34:34:256:4294967295] 192.168.202.29@tcp LPNI seq info [3:3:8:4294967295] 00000400:00000200:2.0:1788844931.725618:0:70072:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:2.0:1788844931.725625:0:70072:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : PUT try# 0 00000800:00000200:2.0:1788844931.725631:0:70072:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.202.29@tcp 00000800:00000010:2.0:1788844931.725636:0:70072:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000f3337948. 00000800:00000200:2.0:1788844931.725644:0:70072:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001785d6c] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:2.0:1788844931.725654:0:70072:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000200:2.0:1788844931.725659:0:70072:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f3337948 type 1, nob 160 niov 1 nkiov 1 00000400:00000200:2.0:1788844931.725685:0:70072:0:(peer.c:4164:lnet_peer_send_push()) peer 192.168.202.29@tcp 00000400:00000200:2.0:1788844931.725687:0:70072:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.202.29@tcp(00000000d558236e) state 0x541 rc 0 00000800:00000200:3.0:1788844931.725943:0:70082:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788844931.725948:0:70082:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000f3337948. 00000400:00000200:3.0:1788844931.725952:0:70082:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788844931.725957:0:70082:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: PUT: OK 00000400:00000200:3.0:1788844931.725962:0:70082:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1788844931.725965:0:70082:0:(peer.c:3241:lnet_discovery_event_send()) Push Send to 192.168.202.29@tcp: 0 00000400:00000200:3.0:1788844931.725987:0:70082:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : PUT - for me 00000400:00000200:3.0:1788844931.725997:0:70082:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.29@tcp of length 64 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1788844931.726004:0:70082:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 0 from 12345-192.168.202.29@tcp of length 64/64 into md 0x5 [1] + 0 00000400:00000010:3.0:1788844931.726009:0:70082:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000078ad9a61. 00000400:00000200:3.0:1788844931.726012:0:70082:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000090a3025 00000400:00000200:3.0:1788844931.726021:0:70082:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788844931.726024:0:70082:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: PUT: OK 00000400:00000200:3.0:1788844931.726028:0:70082:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000090a3025 00000400:00000010:3.0:1788844931.726030:0:70082:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000090a3025. 00000400:00000200:3.0:1788844931.726034:0:70082:0:(api-ni.c:2393:lnet_push_target_event_handler()) type 2 status 0 unlinked 1 00000400:00000200:3.0:1788844931.726039:0:70082:0:(peer.c:2778:lnet_peer_push_event()) peer 192.168.202.29@tcp(00000000d558236e) is MR 00000400:00000010:3.0:1788844931.726043:0:70082:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 73 at 00000000856d004e. 00000400:00000200:3.0:1788844931.726046:0:70082:0:(peer.c:2852:lnet_peer_push_event()) Received Push 192.168.202.29@tcp 12 00000400:00000200:3.0:1788844931.726049:0:70082:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.202.29@tcp: -114 00000400:00000200:3.0:1788844931.726061:0:70082:0:(lib-md.c:96:lnet_cpt_of_md()) Couldn't resolve first page of md 0000000000000000 with offset 0 00000400:00000200:3.0:1788844931.726066:0:70082:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.29@tcp to MR: 192.168.202.29@tcp local destination 00000400:00000200:3.0:1788844931.726070:0:70082:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [35:35:256:4294967295] 192.168.202.29@tcp LPNI seq info [4:4:8:4294967295] 00000400:00000200:3.0:1788844931.726077:0:70082:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) 192.168.202.29@tcp : ACK try# 0 00000800:00000200:3.0:1788844931.726091:0:70082:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 1 frags to 12345-192.168.202.29@tcp 00000800:00000010:3.0:1788844931.726094:0:70082:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000d9d7246b. 00000800:00000200:3.0:1788844931.726098:0:70082:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001785d6c] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:3.0:1788844931.726103:0:70082:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000200:3.0:1788844931.726106:0:70082:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000d9d7246b type 0, nob 96 niov 1 nkiov 0 00000800:00000200:3.0:1788844931.726139:0:70082:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788844931.726142:0:70082:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000d9d7246b. 00000400:00000200:3.0:1788844931.726144:0:70082:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788844931.726147:0:70082:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: ACK: OK 00000400:00000200:3.0:1788844931.726159:0:70082:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp) <- 192.168.202.29@tcp : ACK - for me 00000400:00000200:3.0:1788844931.726167:0:70082:0:(lib-move.c:4539:lnet_parse_ack()) 192.168.202.29@tcp: ACK from 12345-192.168.202.29@tcp into md 0x15 00000400:00000200:3.0:1788844931.726170:0:70082:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788844931.726172:0:70082:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: ACK: OK 00000400:00000200:3.0:1788844931.726175:0:70082:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001ebb8e73 00000400:00000010:3.0:1788844931.726176:0:70082:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000001ebb8e73. 00000400:00000200:3.0:1788844931.726179:0:70082:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 4 00000400:00000200:3.0:1788844931.726182:0:70082:0:(peer.c:3007:lnet_discovery_event_ack()) peer 192.168.202.29@tcp ev->status 0 00000400:00000200:3.0:1788844931.726184:0:70082:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.202.29@tcp: -114 00000400:00000200:2.0:1788844931.726246:0:70072:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000010:2.0:1788844931.726251:0:70072:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c8baaf2a. 00000400:00000010:2.0:1788844931.726255:0:70072:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b903ddb3. 00000400:00000200:2.0:1788844931.726259:0:70072:0:(api-ni.c:2383:lnet_push_target_post()) posted push target 0000000023f07e03 00000400:00000200:2.0:1788844931.726273:0:70072:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.202.29@tcp(00000000d558236e) state 0xc1 00000400:00000010:2.0:1788844931.726277:0:70072:0:(peer.c:3474:lnet_peer_merge_data()) alloc '(curnis)': 60 at 00000000d5dded7f. 00000400:00000010:2.0:1788844931.726279:0:70072:0:(peer.c:3475:lnet_peer_merge_data()) alloc '(addnis)': 72 at 00000000d3a31ee7. 00000400:00000010:2.0:1788844931.726281:0:70072:0:(peer.c:3476:lnet_peer_merge_data()) alloc '(delnis)': 60 at 00000000ebd2e1d2. 00000400:00000010:2.0:1788844931.726284:0:70072:0:(peer.c:3623:lnet_peer_merge_data()) kfreed 'curnis': 60 at 00000000d5dded7f. 00000400:00000010:2.0:1788844931.726286:0:70072:0:(peer.c:3624:lnet_peer_merge_data()) kfreed 'addnis': 72 at 00000000d3a31ee7. 00000400:00000010:2.0:1788844931.726288:0:70072:0:(peer.c:3625:lnet_peer_merge_data()) kfreed 'delnis': 60 at 00000000ebd2e1d2. 00000400:00000010:2.0:1788844931.726289:0:70072:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 73 at 00000000856d004e. 00000400:00000200:2.0:1788844931.726292:0:70072:0:(peer.c:3628:lnet_peer_merge_data()) peer 192.168.202.29@tcp (00000000d558236e): 0 00000400:00000200:2.0:1788844931.726294:0:70072:0:(peer.c:3940:lnet_peer_data_present()) peer 192.168.202.29@tcp(00000000d558236e): 0. state = 0x141 00000400:00000200:2.0:1788844931.726297:0:70072:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.202.29@tcp(00000000d558236e) state 0x141 rc 1 00000400:00000200:2.0:1788844931.726306:0:70072:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.202.29@tcp(00000000d558236e) state 0x141 00000400:00000200:2.0:1788844931.726309:0:70072:0:(peer.c:4082:lnet_peer_discovered()) peer 192.168.202.29@tcp 00000400:00000200:2.0:1788844931.726310:0:70072:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.202.29@tcp(00000000d558236e) state 0x111 rc 0 00000400:00000200:2.0:1788844931.726313:0:70072:0:(peer.c:2629:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 192.168.202.29@tcp 00000400:00000200:3.0:1788844931.726425:0:70322:0:(peer.c:2984:lnet_discover_peer_locked()) peer 192.168.202.29@tcp NID 192.168.202.29@tcp: 0. discovery complete 00000001:02000400:1.0:1788844931.736493:0:70323:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 0 00000400:00000010:3.0:1788844931.805454:0:70326:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 12 at 00000000b73d5114. 00000400:00000010:3.0:1788844931.805469:0:70326:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 12 at 00000000b73d5114. 00000001:02000400:1.0:1788844931.834929:0:70328:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set max_recovery_ping_interval 4 00000001:02000400:2.0:1788844931.869988:0:70330:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp -d *@tcp -m GET -r 1 -e remote_error 00000400:00000200:3.0:1788844931.886403:0:70331:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844931.886411:0:70331:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844931.886422:0:70331:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp 00000400:00000200:3.0:1788844931.886424:0:70331:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844931.886426:0:70331:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp 00000400:00000200:3.0:1788844931.886428:0:70331:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844931.886430:0:70331:0:(api-ni.c:3207:nla_strnid()) -> 00000400:00000200:3.0:1788844931.886431:0:70331:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844931.886433:0:70331:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844931.886434:0:70331:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844931.886436:0:70331:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844931.886437:0:70331:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844931.886438:0:70331:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:3.0:1788844931.886445:0:70331:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000010:3.0:1788844931.886449:0:70331:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 000000007022887f. 00000400:00000200:3.0:1788844931.886456:0:70331:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp, dst 255.255.255.255@tcp, rate 1, interval 0 00000001:02000400:1.0:1788844931.898422:0:70332:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp1 -d *@tcp1 -m GET -r 1 -e remote_error 00000400:00000200:2.0:1788844931.922581:0:70333:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844931.922592:0:70333:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844931.922608:0:70333:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1 00000400:00000200:2.0:1788844931.922610:0:70333:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844931.922613:0:70333:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1 00000400:00000200:2.0:1788844931.922615:0:70333:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844931.922617:0:70333:0:(api-ni.c:3207:nla_strnid()) -> 00000400:00000200:2.0:1788844931.922618:0:70333:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844931.922619:0:70333:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844931.922621:0:70333:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844931.922622:0:70333:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844931.922623:0:70333:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844931.922625:0:70333:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:2.0:1788844931.922626:0:70333:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000010:2.0:1788844931.922631:0:70333:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 0000000092fd5623. 00000400:00000200:2.0:1788844931.922996:0:70333:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp1, dst 255.255.255.255@tcp1, rate 1, interval 0 00000001:02000400:1.0:1788844931.934731:0:70334:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net_drop add -s *@tcp -d *@tcp -r 1 00000400:00000200:0.0:1788844931.957423:0:70335:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844931.957433:0:70335:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844931.957445:0:70335:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp 00000400:00000200:0.0:1788844931.957446:0:70335:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844931.957448:0:70335:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp 00000400:00000200:0.0:1788844931.957449:0:70335:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844931.957459:0:70335:0:(api-ni.c:3207:nla_strnid()) -> 00000400:00000200:0.0:1788844931.957460:0:70335:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844931.957461:0:70335:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844931.957462:0:70335:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844931.957464:0:70335:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844931.957465:0:70335:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844931.957466:0:70335:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844931.957468:0:70335:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000010:0.0:1788844931.957472:0:70335:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 0000000061c20f74. 00000400:00000200:0.0:1788844931.957479:0:70335:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp, dst 255.255.255.255@tcp, rate 1, interval 0 00000001:02000400:0.0:1788844931.972114:0:70336:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp1 -d *@tcp1 -r 1 00000400:00000200:0.0:1788844931.993728:0:70337:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844931.993739:0:70337:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844931.993755:0:70337:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1 00000400:00000200:0.0:1788844931.993756:0:70337:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844931.993759:0:70337:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1 00000400:00000200:0.0:1788844931.993760:0:70337:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844931.993763:0:70337:0:(api-ni.c:3207:nla_strnid()) -> 00000400:00000200:0.0:1788844931.993764:0:70337:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844931.993765:0:70337:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844931.993766:0:70337:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844931.993768:0:70337:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844931.993769:0:70337:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844931.993771:0:70337:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000200:0.0:1788844931.993772:0:70337:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5 00000400:00000010:0.0:1788844931.993777:0:70337:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 0000000080d3ffc4. 00000400:00000200:0.0:1788844931.993871:0:70337:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp1, dst 255.255.255.255@tcp1, rate 1, interval 0 00000001:02000400:1.0:1788844932.006539:0:70338:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --nid 192.168.202.29@tcp --health 0 00000400:00000200:2.0:1788844932.026189:0:70339:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.202.29@tcp added to recovery queue. ping count: 0 next ping: 1496 last alive: 1495 health: 0 00000400:00000010:0.0:1788844932.078227:0:70345:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 000000002bfd01db. 00000400:00000010:0.0:1788844932.078334:0:70345:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 000000002bfd01db. 00000400:00000010:0.0:1788844933.171653:0:70356:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 000000002bfd01db. 00000400:00000010:0.0:1788844933.171738:0:70356:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 000000002bfd01db. 00000400:00000010:1.0:1788844933.210934:0:70071:0:(lib-move.c:3721:lnet_recover_peer_nis()) alloc '(ev_info)': 24 at 000000009ea43f46. 00000400:00000010:1.0:1788844933.210950:0:70071:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000003f62c563. 00000400:00000010:1.0:1788844933.210957:0:70071:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000008617e501. 00000400:00000010:1.0:1788844933.210965:0:70071:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000032a541b4. 00000400:00000200:1.0:1788844933.210987:0:70071:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.29@tcp 00000400:00000200:1.0:1788844933.211001:0:70071:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.29@tcp local destination 00000400:00000200:1.0:1788844933.211008:0:70071:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.29@tcp 00000400:00000200:1.0:1788844933.211012:0:70071:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [36:36:256:4294967295] 192.168.202.29@tcp LPNI seq info [5:5:8:4294967295] 00000400:00000200:1.0:1788844933.211020:0:70071:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:1.0:1788844933.211028:0:70071:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : GET try# 0 00000800:00000200:1.0:1788844933.211033:0:70071:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.29@tcp 00000800:00000010:1.0:1788844933.211038:0:70071:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f3ee3836. 00000800:00000200:1.0:1788844933.211045:0:70071:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001785d6c] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:1.0:1788844933.211054:0:70071:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000200:1.0:1788844933.211057:0:70071:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f3ee3836 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1788844933.211077:0:70071:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.202.29@tcp added to recovery queue. ping count: 1 next ping: 1498 last alive: 1495 health: 0 00000400:00000200:3.0:1788844933.211147:0:70082:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 1 00000400:00000200:3.0:1788844933.211154:0:70082:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.202.29@tcp(192.168.202.29@tcp)->dst 192.168.202.29@tcp: GET simulate health error: REMOTE_ERROR 00000800:02000400:3.0:1788844933.211157:0:70082:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.202.29: rc = -22 00000800:00000200:3.0:1788844933.224670:0:70082:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [000000006ade1324] Error -22 on write to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000010:3.0:1788844933.224682:0:70082:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000f3ee3836. 00000400:00000200:3.0:1788844933.224687:0:70082:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788844933.224693:0:70082:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1788844933.224702:0:70082:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5 00000400:00000200:3.0:1788844933.224705:0:70082:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.29@tcp recovery message sent unsuccessfully:-5 00000400:00000200:3.0:1788844933.224711:0:70082:0:(lib-move.c:4104:lnet_handle_recovery_reply()) peer NI (192.168.202.29@tcp) recovery failed with -5 00000400:00000200:1.0:1788844934.232344:0:70071:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000008617e501 00000400:00000010:1.0:1788844934.232360:0:70071:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000008617e501. 00000400:00000200:1.0:1788844934.232371:0:70071:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0 00000400:00000200:1.0:1788844934.232380:0:70071:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.202.29@tcp recovery ping unlinked 00000400:00000010:1.0:1788844934.232388:0:70071:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000009ea43f46. 00000400:00000010:1.0:1788844934.232395:0:70071:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000003f62c563. 00000400:00000010:1.0:1788844934.232407:0:70071:0:(lib-move.c:3721:lnet_recover_peer_nis()) alloc '(ev_info)': 24 at 000000009ea43f46. 00000400:00000010:1.0:1788844934.232412:0:70071:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000003f62c563. 00000400:00000010:1.0:1788844934.232417:0:70071:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000008617e501. 00000400:00000010:1.0:1788844934.232425:0:70071:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000000300d813. 00000400:00000200:1.0:1788844934.232429:0:70071:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.29@tcp 00000400:00000200:1.0:1788844934.232442:0:70071:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.29@tcp local destination 00000400:00000200:1.0:1788844934.232451:0:70071:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.29@tcp 00000400:00000200:1.0:1788844934.232455:0:70071:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [37:37:256:4294967295] 192.168.202.29@tcp LPNI seq info [6:6:8:4294967295] 00000400:00000200:1.0:1788844934.232464:0:70071:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:1.0:1788844934.232471:0:70071:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : GET try# 0 00000800:00000200:1.0:1788844934.232478:0:70071:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.29@tcp 00000800:00000010:1.0:1788844934.232484:0:70071:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000d5a0c195. 00000800:00000200:1.0:1788844934.232508:0:70071:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001785d6c] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:1.0:1788844934.232530:0:70071:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000200:1.0:1788844934.232537:0:70071:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000d5a0c195 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1788844934.232562:0:70071:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.202.29@tcp added to recovery queue. ping count: 2 next ping: 1502 last alive: 1495 health: 0 00000400:00000200:0.0:1788844934.232663:0:70083:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 2 00000400:00000200:0.0:1788844934.232673:0:70083:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.202.29@tcp(192.168.202.29@tcp)->dst 192.168.202.29@tcp: GET simulate health error: REMOTE_ERROR 00000800:02000400:0.0:1788844934.232677:0:70083:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.202.29: rc = -22 00000800:00000200:0.0:1788844934.232683:0:70083:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [00000000867cf3f8] Error -22 on write to 12345-192.168.202.29@tcp ip 192.168.202.29:1023 00000800:00000010:0.0:1788844934.232688:0:70083:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000d5a0c195. 00000400:00000200:0.0:1788844934.232695:0:70083:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788844934.232715:0:70083:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: GET: REMOTE_ERROR 00000400:00000200:0.0:1788844934.232722:0:70083:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5 00000400:00000200:0.0:1788844934.232725:0:70083:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.29@tcp recovery message sent unsuccessfully:-5 00000400:00000200:0.0:1788844934.232731:0:70083:0:(lib-move.c:4104:lnet_handle_recovery_reply()) peer NI (192.168.202.29@tcp) recovery failed with -5 00000400:00000010:0.0:1788844934.381924:0:70368:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 000000002bfd01db. 00000400:00000010:0.0:1788844934.382045:0:70368:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 000000002bfd01db. 00000400:00000200:1.0:1788844935.255927:0:70071:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000008617e501 00000400:00000010:1.0:1788844935.255946:0:70071:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000008617e501. 00000400:00000200:1.0:1788844935.255957:0:70071:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0 00000400:00000200:1.0:1788844935.256391:0:70071:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.202.29@tcp recovery ping unlinked 00000400:00000010:1.0:1788844935.256402:0:70071:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000009ea43f46. 00000400:00000010:1.0:1788844935.256411:0:70071:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000003f62c563. 00000400:00000010:0.0:1788844935.602708:0:70379:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 000000002bfd01db. 00000400:00000010:0.0:1788844935.602846:0:70379:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 000000002bfd01db. 00000400:00000010:1.0:1788844936.760561:0:70390:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 0000000011597222. 00000400:00000010:1.0:1788844936.760667:0:70390:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 0000000011597222. 00000400:00000010:2.0:1788844937.929198:0:70396:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 00000000db156cca. 00000400:00000010:2.0:1788844937.929326:0:70396:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 00000000db156cca. 00000400:00000010:2.0:1788844938.334903:0:70071:0:(lib-move.c:3721:lnet_recover_peer_nis()) alloc '(ev_info)': 24 at 00000000d72a2ad2. 00000400:00000010:2.0:1788844938.334924:0:70071:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000f6efb2b6. 00000400:00000010:2.0:1788844938.334933:0:70071:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000002dfd888. 00000400:00000010:2.0:1788844938.334945:0:70071:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000eebb5301. 00000400:00000200:2.0:1788844938.334956:0:70071:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.29@tcp 00000400:00000200:2.0:1788844938.334973:0:70071:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.29@tcp local destination 00000400:00000200:2.0:1788844938.334981:0:70071:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.29@tcp 00000400:00000200:2.0:1788844938.334986:0:70071:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.29@tcp NI seq info: [38:38:256:4294967295] 192.168.202.29@tcp LPNI seq info [7:7:8:4294967295] 00000400:00000200:2.0:1788844938.334994:0:70071:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.29@tcp 00000400:00000200:2.0:1788844938.335003:0:70071:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.29@tcp(192.168.202.29@tcp:) -> 192.168.202.29@tcp(192.168.202.29@tcp:192.168.202.29@tcp) : GET try# 0 00000800:00000200:2.0:1788844938.335022:0:70071:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.29@tcp 00000800:00000010:2.0:1788844938.335027:0:70071:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000007ace9e47. 00000800:00000200:2.0:1788844938.335035:0:70071:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001785d6c] -> 12345-192.168.202.29@tcp (6) 00000800:00000200:2.0:1788844938.335045:0:70071:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000200:2.0:1788844938.335050:0:70071:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000007ace9e47 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:2.0:1788844938.335069:0:70071:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.202.29@tcp added to recovery queue. ping count: 3 next ping: 1506 last alive: 1495 health: 0 00000400:00000200:0.0:1788844938.335149:0:70081:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 3 00000400:00000200:0.0:1788844938.335155:0:70081:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.202.29@tcp(192.168.202.29@tcp)->dst 192.168.202.29@tcp: GET simulate health error: REMOTE_ERROR 00000800:02000400:0.0:1788844938.335159:0:70081:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.202.29: rc = -22 00000800:00000200:0.0:1788844938.335164:0:70081:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [000000006ade1324] Error -22 on write to 12345-192.168.202.29@tcp ip 192.168.202.29:988 00000800:00000010:0.0:1788844938.335168:0:70081:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000007ace9e47. 00000400:00000200:0.0:1788844938.335173:0:70081:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788844938.335177:0:70081:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.29@tcp->192.168.202.29@tcp: GET: REMOTE_ERROR 00000400:00000200:0.0:1788844938.335185:0:70081:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5 00000400:00000200:0.0:1788844938.335188:0:70081:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.29@tcp recovery message sent unsuccessfully:-5 00000400:00000200:0.0:1788844938.335192:0:70081:0:(lib-move.c:4104:lnet_handle_recovery_reply()) peer NI (192.168.202.29@tcp) recovery failed with -5 00000400:00000200:1.0:1788844939.353017:0:70071:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000002dfd888 00000400:00000010:1.0:1788844939.353034:0:70071:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 0000000002dfd888. 00000400:00000200:1.0:1788844939.353042:0:70071:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0 00000400:00000200:1.0:1788844939.353053:0:70071:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.202.29@tcp recovery ping unlinked 00000400:00000010:1.0:1788844939.353062:0:70071:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000d72a2ad2. 00000400:00000010:1.0:1788844939.353071:0:70071:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000f6efb2b6. 00000001:02000400:2.0:1788844939.939004:0:70523:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_211: @@@@@@ FAIL: Unexpected ping count Debug log: 4726 lines, 4726 kept, 0 dropped, 0 bad.