00000001:02000400:0.0F:1787866183.300568:0:67952:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 210: Local NI recovery checks ======== 17:29:41 (1787866181)
00000400:00000010:3.0F:1787866183.787517:0:67986:0:(lib-cpt.c:67:cfs_cpt_table_alloc()) alloc '(cptab)': 56 at 00000000deb25d37.
00000400:00000010:3.0:1787866183.787537:0:67986:0:(lib-cpt.c:76:cfs_cpt_table_alloc()) alloc '(cptab->ctb_nodemask)': 128 at 0000000000ae0ad0.
00000400:00000010:3.0:1787866183.787541:0:67986:0:(lib-cpt.c:80:cfs_cpt_table_alloc()) alloc '(cptab->ctb_cpu2cpt)': 16 at 00000000562b4bc7.
00000400:00000010:3.0:1787866183.787544:0:67986:0:(lib-cpt.c:87:cfs_cpt_table_alloc()) alloc '(cptab->ctb_node2cpt)': 4 at 000000004a1675f1.
00000400:00000010:3.0:1787866183.787547:0:67986:0:(lib-cpt.c:94:cfs_cpt_table_alloc()) alloc '(cptab->ctb_parts)': 32 at 000000005e5911e8.
00000400:00000010:3.0:1787866183.787552:0:67986:0:(lib-cpt.c:106:cfs_cpt_table_alloc()) alloc '(part->cpt_nodemask)': 128 at 00000000acb6c7bc.
00000400:00000010:3.0:1787866183.787554:0:67986:0:(lib-cpt.c:110:cfs_cpt_table_alloc()) alloc '(part->cpt_distance)': 4 at 00000000e952e04a.
00000400:00000010:3.0:1787866184.583803:0:67986:0:(lock.c:52:cfs_percpt_lock_create()) alloc '(pcl)': 24 at 00000000df958599.
00000400:00000010:3.0:1787866184.583815:0:67986:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000b3915b1f.
00000400:00000010:3.0:1787866184.583821:0:67986:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000061d2b9ee.
00000400:00000010:3.0:1787866184.583824:0:67986:0:(lock.c:52:cfs_percpt_lock_create()) alloc '(pcl)': 24 at 000000007d44a977.
00000400:00000010:3.0:1787866184.583826:0:67986:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000abc8e8a8.
00000400:00000010:3.0:1787866184.583827:0:67986:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000dae8013c.
00000400:00000200:3.0:1787866184.583963:0:67986:0:(api-ni.c:978:lnet_register_lnd()) lo LND registered
00000400:00000200:2.0F:1787866184.751186:0:68002:0:(api-ni.c:978:lnet_register_lnd()) tcp LND registered
00000001:02000400:1.0F:1787866184.785562:0:68004:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure
00000400:00000010:3.0:1787866184.816366:0:68005:0:(api-ni.c:676:lnet_create_remote_nets_table()) alloc '(hash)': 1024 at 00000000d21aed04.
00000400:00000010:3.0:1787866184.816377:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000009f83563a.
00000400:00000010:3.0:1787866184.816382:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000eb7a7e8e.
00000400:00000010:3.0:1787866184.816384:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000037e66e1d.
00000400:00000010:3.0:1787866184.816386:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000cffd9ef1.
00000400:00000010:3.0:1787866184.816392:0:68005:0:(peer.c:109:lnet_peer_tables_create()) alloc '(hash)': 8192 at 0000000066cc6be4.
00000400:00000010:3.0:1787866184.816397:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e4a9e237.
00000400:00000010:3.0:1787866184.816399:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000b23e87d0.
00000400:00000010:3.0:1787866184.816402:0:68005:0:(lib-msg.c:1338:lnet_msg_container_setup()) alloc '(container->msc_finalizers)': 32 at 0000000080be1e14.
00000400:00000010:3.0:1787866184.816405:0:68005:0:(lib-msg.c:1348:lnet_msg_container_setup()) alloc '(container->msc_resenders)': 32 at 000000006efbdcd1.
00000400:00000010:3.0:1787866184.816492:0:68005:0:(api-ni.c:1227:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 00000000dc7691bc.
00000400:00000010:3.0:1787866184.817941:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000eb9b3a86.
00000400:00000010:3.0:1787866184.817952:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d61d6264.
00000400:00000010:3.0:1787866184.818101:0:68005:0:(api-ni.c:1227:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 00000000479850c5.
00000400:00000010:3.0:1787866184.818123:0:68005:0:(lib-ptl.c:864:lnet_portals_create()) alloc '(the_lnet.ln_portals)': 512 at 00000000900df17a.
00000400:00000010:3.0:1787866184.818126:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000049c09a60.
00000400:00000010:3.0:1787866184.818129:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d576df6d.
00000400:00000010:3.0:1787866184.818131:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004494a819.
00000400:00000010:3.0:1787866184.818137:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d94bb90a.
00000400:00000010:3.0:1787866184.818146:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000023ff55ad.
00000400:00000010:3.0:1787866184.818148:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000050988b52.
00000400:00000010:3.0:1787866184.818150:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001e04f513.
00000400:00000010:3.0:1787866184.818155:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f3b8bfe7.
00000400:00000010:3.0:1787866184.818157:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000b4b34503.
00000400:00000010:3.0:1787866184.818159:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002bc5854b.
00000400:00000010:3.0:1787866184.818161:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000cccc9757.
00000400:00000010:3.0:1787866184.818164:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002f504b20.
00000400:00000010:3.0:1787866184.818167:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000c46746d3.
00000400:00000010:3.0:1787866184.818168:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002379337e.
00000400:00000010:3.0:1787866184.818170:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d8180f8a.
00000400:00000010:3.0:1787866184.818174:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000dbb65273.
00000400:00000010:3.0:1787866184.818176:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000086becfe9.
00000400:00000010:3.0:1787866184.818178:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000078b0cb5f.
00000400:00000010:3.0:1787866184.818179:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000cea13fef.
00000400:00000010:3.0:1787866184.818183:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000042dece22.
00000400:00000010:3.0:1787866184.818185:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e0f4b49a.
00000400:00000010:3.0:1787866184.818186:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a513f0c7.
00000400:00000010:3.0:1787866184.818188:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008be1229f.
00000400:00000010:3.0:1787866184.818191:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ede22331.
00000400:00000010:3.0:1787866184.818193:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000009f0e7ba3.
00000400:00000010:3.0:1787866184.818195:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000022f4e0e5.
00000400:00000010:3.0:1787866184.818197:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ccee0bbb.
00000400:00000010:3.0:1787866184.818201:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000874db7d2.
00000400:00000010:3.0:1787866184.818203:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000008d8322ba.
00000400:00000010:3.0:1787866184.818205:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000009a537483.
00000400:00000010:3.0:1787866184.818206:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005cf25d6b.
00000400:00000010:3.0:1787866184.818210:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ed353615.
00000400:00000010:3.0:1787866184.818213:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a5aed5b6.
00000400:00000010:3.0:1787866184.818214:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000067279915.
00000400:00000010:3.0:1787866184.818216:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b81fa609.
00000400:00000010:3.0:1787866184.818219:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006e575a31.
00000400:00000010:3.0:1787866184.818222:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e3cbe3fc.
00000400:00000010:3.0:1787866184.818228:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000b1b9e545.
00000400:00000010:3.0:1787866184.818230:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000de939f20.
00000400:00000010:3.0:1787866184.818234:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000091bd4b17.
00000400:00000010:3.0:1787866184.818236:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000007154a7e2.
00000400:00000010:3.0:1787866184.818237:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000006f39c091.
00000400:00000010:3.0:1787866184.818239:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000067094ef7.
00000400:00000010:3.0:1787866184.818242:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f35d978d.
00000400:00000010:3.0:1787866184.818245:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000024d8becd.
00000400:00000010:3.0:1787866184.818246:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000084ec0e37.
00000400:00000010:3.0:1787866184.818248:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003d968886.
00000400:00000010:3.0:1787866184.818254:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004da4d315.
00000400:00000010:3.0:1787866184.818257:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000003ac074f2.
00000400:00000010:3.0:1787866184.818258:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000436d5143.
00000400:00000010:3.0:1787866184.818260:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000066d309d2.
00000400:00000010:3.0:1787866184.818264:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007666a5df.
00000400:00000010:3.0:1787866184.818266:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000001a570e11.
00000400:00000010:3.0:1787866184.818268:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a1903a0c.
00000400:00000010:3.0:1787866184.818270:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004e96b557.
00000400:00000010:3.0:1787866184.818274:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000007654107.
00000400:00000010:3.0:1787866184.818276:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000f615b3f7.
00000400:00000010:3.0:1787866184.818278:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001457018c.
00000400:00000010:3.0:1787866184.818279:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004dc61355.
00000400:00000010:3.0:1787866184.818283:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000eec38694.
00000400:00000010:3.0:1787866184.818286:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000007545239e.
00000400:00000010:3.0:1787866184.818287:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000092c1d6d3.
00000400:00000010:3.0:1787866184.818289:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e6636fb0.
00000400:00000010:3.0:1787866184.818294:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000af0841f6.
00000400:00000010:3.0:1787866184.818297:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000093c9fcf8.
00000400:00000010:3.0:1787866184.818299:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000081f58040.
00000400:00000010:3.0:1787866184.818300:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005bad2a7b.
00000400:00000010:3.0:1787866184.818304:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000372d9216.
00000400:00000010:3.0:1787866184.818306:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ca528c1f.
00000400:00000010:3.0:1787866184.818313:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000bfd43d4d.
00000400:00000010:3.0:1787866184.818315:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a4963b83.
00000400:00000010:3.0:1787866184.818320:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000066fd5643.
00000400:00000010:3.0:1787866184.818322:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000040d72397.
00000400:00000010:3.0:1787866184.818323:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000c408af56.
00000400:00000010:3.0:1787866184.818325:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b39d4f03.
00000400:00000010:3.0:1787866184.818328:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001fc9d250.
00000400:00000010:3.0:1787866184.818330:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000021f0813b.
00000400:00000010:3.0:1787866184.818332:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000004d0945d9.
00000400:00000010:3.0:1787866184.818334:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009e0d68a0.
00000400:00000010:3.0:1787866184.818338:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ad1ad778.
00000400:00000010:3.0:1787866184.818341:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000056efd22.
00000400:00000010:3.0:1787866184.818343:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000cb4bd87b.
00000400:00000010:3.0:1787866184.818344:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004f995630.
00000400:00000010:3.0:1787866184.818348:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003cb04dca.
00000400:00000010:3.0:1787866184.818351:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000b3612087.
00000400:00000010:3.0:1787866184.818353:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000315ac413.
00000400:00000010:3.0:1787866184.818354:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b2540f13.
00000400:00000010:3.0:1787866184.818358:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ff0fa99f.
00000400:00000010:3.0:1787866184.818361:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000b0bcd302.
00000400:00000010:3.0:1787866184.818362:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e77cd98a.
00000400:00000010:3.0:1787866184.818364:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000069317189.
00000400:00000010:3.0:1787866184.818368:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b355e253.
00000400:00000010:3.0:1787866184.818371:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a8e3b738.
00000400:00000010:3.0:1787866184.818372:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000aeed7957.
00000400:00000010:3.0:1787866184.818374:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e3db74e8.
00000400:00000010:3.0:1787866184.818379:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000061615e9.
00000400:00000010:3.0:1787866184.818381:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000007da5d9b4.
00000400:00000010:3.0:1787866184.818383:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000b3074146.
00000400:00000010:3.0:1787866184.818385:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000072564e3b.
00000400:00000010:3.0:1787866184.818389:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a01446bf.
00000400:00000010:3.0:1787866184.818392:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ab4cbdf5.
00000400:00000010:3.0:1787866184.818401:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000c10d026b.
00000400:00000010:3.0:1787866184.818403:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a710530b.
00000400:00000010:3.0:1787866184.818407:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ec41b373.
00000400:00000010:3.0:1787866184.818410:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e9590d32.
00000400:00000010:3.0:1787866184.818412:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002aa3873b.
00000400:00000010:3.0:1787866184.818414:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000bfac8294.
00000400:00000010:3.0:1787866184.818418:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002bcb5613.
00000400:00000010:3.0:1787866184.818421:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e4649c08.
00000400:00000010:3.0:1787866184.818422:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a0fb2ddc.
00000400:00000010:3.0:1787866184.818424:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009fc95cce.
00000400:00000010:3.0:1787866184.818429:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b5fbf0e5.
00000400:00000010:3.0:1787866184.818432:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000b2a3b6a8.
00000400:00000010:3.0:1787866184.818434:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005a3e6480.
00000400:00000010:3.0:1787866184.818435:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f45699e6.
00000400:00000010:3.0:1787866184.818439:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000956c410a.
00000400:00000010:3.0:1787866184.818441:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000003ad4337c.
00000400:00000010:3.0:1787866184.818442:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000f52ae701.
00000400:00000010:3.0:1787866184.818443:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000dc92e049.
00000400:00000010:3.0:1787866184.818447:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003faedac8.
00000400:00000010:3.0:1787866184.818450:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000012d03674.
00000400:00000010:3.0:1787866184.818452:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000351f65b9.
00000400:00000010:3.0:1787866184.818454:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000020a8a065.
00000400:00000010:3.0:1787866184.818458:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e93507ee.
00000400:00000010:3.0:1787866184.818461:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000009ffd5ad9.
00000400:00000010:3.0:1787866184.818463:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000046a43c45.
00000400:00000010:3.0:1787866184.818465:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000090520b28.
00000400:00000010:3.0:1787866184.818479:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005604167f.
00000400:00000010:3.0:1787866184.818482:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000008787625a.
00000400:00000010:3.0:1787866184.818484:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000077e5c6ec.
00000400:00000010:3.0:1787866184.818486:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009ec4540a.
00000400:00000010:3.0:1787866184.818490:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000049f79d76.
00000400:00000010:3.0:1787866184.818492:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000016ffbbec.
00000400:00000010:3.0:1787866184.818499:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002602338d.
00000400:00000010:3.0:1787866184.818501:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000303fbebc.
00000400:00000010:3.0:1787866184.819173:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000028b9089e.
00000400:00000010:3.0:1787866184.819183:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000070635d03.
00000400:00000010:3.0:1787866184.819186:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000aded3c3a.
00000400:00000010:3.0:1787866184.819188:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000cfcf2778.
00000400:00000010:3.0:1787866184.819205:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000cb480af2.
00000400:00000010:3.0:1787866184.819208:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a3257d5d.
00000400:00000010:3.0:1787866184.819210:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000873b033d.
00000400:00000010:3.0:1787866184.819212:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003a038b54.
00000400:00000010:3.0:1787866184.819215:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a27db30f.
00000400:00000010:3.0:1787866184.819218:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000072de1828.
00000400:00000010:3.0:1787866184.819220:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d6f93c62.
00000400:00000010:3.0:1787866184.819222:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000bf17c8c4.
00000400:00000010:3.0:1787866184.819226:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ce65c75a.
00000400:00000010:3.0:1787866184.819228:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000072029be1.
00000400:00000010:3.0:1787866184.819230:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000685aa72e.
00000400:00000010:3.0:1787866184.819232:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000aaddbc94.
00000400:00000010:3.0:1787866184.819235:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007434b255.
00000400:00000010:3.0:1787866184.819238:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a4918ca0.
00000400:00000010:3.0:1787866184.819240:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e65ff80a.
00000400:00000010:3.0:1787866184.819242:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006a1e06d4.
00000400:00000010:3.0:1787866184.819252:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000076d6cd1c.
00000400:00000010:3.0:1787866184.819256:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ab15aaf2.
00000400:00000010:3.0:1787866184.819258:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000080b4b9f8.
00000400:00000010:3.0:1787866184.819260:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fc4e5ceb.
00000400:00000010:3.0:1787866184.819263:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000015c7a150.
00000400:00000010:3.0:1787866184.819265:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000fd0ac38c.
00000400:00000010:3.0:1787866184.819267:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000000d103696.
00000400:00000010:3.0:1787866184.819269:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f13b8109.
00000400:00000010:3.0:1787866184.819272:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006d892fb3.
00000400:00000010:3.0:1787866184.819275:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000aaa9c056.
00000400:00000010:3.0:1787866184.819283:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000009467422c.
00000400:00000010:3.0:1787866184.819284:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f1ecd397.
00000400:00000010:3.0:1787866184.819288:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000072a8f16f.
00000400:00000010:3.0:1787866184.819291:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000007ae10c44.
00000400:00000010:3.0:1787866184.819292:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000026f6c9c0.
00000400:00000010:3.0:1787866184.819294:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002955af4b.
00000400:00000010:3.0:1787866184.819303:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a3e62e9d.
00000400:00000010:3.0:1787866184.819306:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000be2546b6.
00000400:00000010:3.0:1787866184.819308:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000067d2cebe.
00000400:00000010:3.0:1787866184.819310:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ad0a5617.
00000400:00000010:3.0:1787866184.819314:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000799598b3.
00000400:00000010:3.0:1787866184.819316:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000048d6c5d0.
00000400:00000010:3.0:1787866184.819317:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001e54cbd7.
00000400:00000010:3.0:1787866184.819319:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000036360d9c.
00000400:00000010:3.0:1787866184.819322:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e53a5eee.
00000400:00000010:3.0:1787866184.819325:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000036b3a50e.
00000400:00000010:3.0:1787866184.819326:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005586a200.
00000400:00000010:3.0:1787866184.819328:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f0975345.
00000400:00000010:3.0:1787866184.819331:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000bce067fc.
00000400:00000010:3.0:1787866184.819333:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000375896ff.
00000400:00000010:3.0:1787866184.819335:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ef2cd2ea.
00000400:00000010:3.0:1787866184.819338:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003c4758a4.
00000400:00000010:3.0:1787866184.819347:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000067e117b8.
00000400:00000010:3.0:1787866184.819351:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000012c6c579.
00000400:00000010:3.0:1787866184.819353:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000023cd7e93.
00000400:00000010:3.0:1787866184.819355:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001a01547f.
00000400:00000010:3.0:1787866184.819359:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003dca47e3.
00000400:00000010:3.0:1787866184.819361:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000013205a57.
00000400:00000010:3.0:1787866184.819363:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000008876f365.
00000400:00000010:3.0:1787866184.819365:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000639234ad.
00000400:00000010:3.0:1787866184.819368:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000052006a4e.
00000400:00000010:3.0:1787866184.819370:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000007bf2fb4c.
00000400:00000010:3.0:1787866184.819378:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000006af87bde.
00000400:00000010:3.0:1787866184.819380:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000015c73426.
00000400:00000010:3.0:1787866184.819384:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006ce923da.
00000400:00000010:3.0:1787866184.819386:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000094312786.
00000400:00000010:3.0:1787866184.819388:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000065590823.
00000400:00000010:3.0:1787866184.819389:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004d9a224a.
00000400:00000010:3.0:1787866184.819399:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ff320cb3.
00000400:00000010:3.0:1787866184.819402:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000aa387aa5.
00000400:00000010:3.0:1787866184.819404:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001c3ad77c.
00000400:00000010:3.0:1787866184.819406:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001d9cf5b7.
00000400:00000010:3.0:1787866184.819409:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000010775746.
00000400:00000010:3.0:1787866184.819411:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000088e7fc31.
00000400:00000010:3.0:1787866184.819413:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000000c3faf23.
00000400:00000010:3.0:1787866184.819415:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000a23be36.
00000400:00000010:3.0:1787866184.819418:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006fe7da78.
00000400:00000010:3.0:1787866184.819421:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000c2ec0b30.
00000400:00000010:3.0:1787866184.819422:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e7f6ed49.
00000400:00000010:3.0:1787866184.819424:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000aab07357.
00000400:00000010:3.0:1787866184.819428:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f68aaa16.
00000400:00000010:3.0:1787866184.819430:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ae718185.
00000400:00000010:3.0:1787866184.819432:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ea68bc5b.
00000400:00000010:3.0:1787866184.819433:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003f0bdf4c.
00000400:00000010:3.0:1787866184.819443:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000eff1234f.
00000400:00000010:3.0:1787866184.819446:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000002d74340f.
00000400:00000010:3.0:1787866184.819448:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000f2e30eec.
00000400:00000010:3.0:1787866184.819451:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000874a2026.
00000400:00000010:3.0:1787866184.819455:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000000b3665f6.
00000400:00000010:3.0:1787866184.819458:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000009dea2467.
00000400:00000010:3.0:1787866184.819459:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000fccb4c56.
00000400:00000010:3.0:1787866184.819461:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000006ca9eb8.
00000400:00000010:3.0:1787866184.819464:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003506a16b.
00000400:00000010:3.0:1787866184.819466:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000019d8f1d6.
00000400:00000010:3.0:1787866184.819482:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000062c72d1a.
00000400:00000010:3.0:1787866184.819484:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f9cb8dba.
00000400:00000010:3.0:1787866184.819487:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000000ee9133.
00000400:00000010:3.0:1787866184.819490:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000010a6e38f.
00000400:00000010:3.0:1787866184.819492:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d78759ab.
00000400:00000010:3.0:1787866184.819493:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000dfd99f4c.
00000400:00000010:3.0:1787866184.819502:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e4747cb8.
00000400:00000010:3.0:1787866184.819520:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000005e1ca33c.
00000400:00000010:3.0:1787866184.819524:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000006ef8a308.
00000400:00000010:3.0:1787866184.819526:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007aed3052.
00000400:00000010:3.0:1787866184.819530:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f8e9a5a4.
00000400:00000010:3.0:1787866184.819532:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000000f2b7a98.
00000400:00000010:3.0:1787866184.819534:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000cd87e7a3.
00000400:00000010:3.0:1787866184.819536:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000be3d3d27.
00000400:00000010:3.0:1787866184.819539:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000079ea313f.
00000400:00000010:3.0:1787866184.819541:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e6757c30.
00000400:00000010:3.0:1787866184.819543:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000b0e1e2cf.
00000400:00000010:3.0:1787866184.819544:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009c022a70.
00000400:00000010:3.0:1787866184.819548:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000088e78596.
00000400:00000010:3.0:1787866184.819550:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000b27f351f.
00000400:00000010:3.0:1787866184.819552:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005c3c6f51.
00000400:00000010:3.0:1787866184.819553:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f9a8cb83.
00000400:00000010:3.0:1787866184.819563:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005083cb8a.
00000400:00000010:3.0:1787866184.819567:0:68005:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000087818236.
00000400:00000010:3.0:1787866184.819569:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000003e32eaa7.
00000400:00000010:3.0:1787866184.819571:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000df0a1b16.
00000400:00000010:3.0:1787866184.819574:0:68005:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008198bbcd.
00000400:00000010:3.0:1787866184.819577:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000f23e1bfd.
00000400:00000010:3.0:1787866184.819579:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000037f27f8a.
00000400:00000010:3.0:1787866184.819583:0:68005:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 00000000ea694c1b.
00000400:00000010:3.0:1787866184.819587:0:68005:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 552 at 0000000092bb6b37.
00000400:00000010:3.0:1787866184.819589:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d74fa375.
00000400:00000010:3.0:1787866184.819591:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ca7f81c8.
00000400:00000010:3.0:1787866184.819597:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000000b98e9f.
00000400:00000010:3.0:1787866184.819598:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f6653601.
00000400:00000010:3.0:1787866184.819612:0:68005:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 41 at 0000000055425607.
00000400:00000010:3.0:1787866184.819618:0:68005:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000084f4344e.
00000400:00000010:3.0:1787866184.819632:0:68005:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000021aec6f6.
00000400:00000200:3.0:1787866184.819642:0:68005:0:(lib-ptl.c:918:LNetSetLazyPortal()) Setting portal 0 lazy
00000400:00000010:3.0:1787866184.819646:0:68005:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000ffa05391.
00000400:00000010:3.0:1787866184.819649:0:68005:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b3af47a7.
00000400:00000010:3.0:1787866184.819651:0:68005:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a3edd4bd.
00000400:00000200:3.0:1787866184.819653:0:68005:0:(api-ni.c:2383:lnet_push_target_post()) posted push target 00000000ffa05391
00000400:00000200:3.0:1787866184.821125:0:68005:0:(api-ni.c:2340:lnet_push_target_resize()) nbytes 272 success
00000400:00000010:3.0:1787866184.821136:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ca6fac8d.
00000400:00000010:3.0:1787866184.821142:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000039e568ed.
00000400:00000010:3.0:1787866184.821144:0:68005:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000007213af63.
00000400:00000010:3.0:1787866184.821146:0:68005:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000075b49a8d.
00000400:00000200:0.0:1787866184.823819:0:68005:0:(peer.c:4457:lnet_peer_discovery_start()) discovery start: 0
00000400:00000200:3.0:1787866184.825315:0:68007:0:(peer.c:4283:lnet_peer_discovery()) started
00000400:00000010:0.0:1787866184.837296:0:68008:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000912c74c3.
00000400:00000010:0.0:1787866184.837322:0:68008:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000912c74c3.
00000400:00000010:0.0:1787866184.837615:0:68008:0:(api-ni.c:7409:lnet_route_show_start()) alloc '(rlist)': 16 at 00000000cc26e41d.
00000400:00000010:0.0:1787866184.837632:0:68008:0:(api-ni.c:7309:lnet_route_show_done()) kfreed 'rlist': 16 at 00000000cc26e41d.
00000400:00000010:0.0:1787866184.838041:0:68008:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 236 at 0000000050d5d605.
00000400:00000010:0.0:1787866184.838050:0:68008:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 236 at 0000000050d5d605.
00000400:00000010:0.0:1787866184.838222:0:68008:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 00000000cc26e41d.
00000400:00000010:0.0:1787866184.838236:0:68008:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 00000000cc26e41d.
00000400:00000010:0.0:1787866184.838307:0:68008:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 12 at 00000000cc26e41d.
00000400:00000010:0.0:1787866184.838312:0:68008:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 12 at 00000000cc26e41d.
00000400:00000010:0.0:1787866184.839436:0:68008:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 32 at 000000004742348e.
00000001:00000200:0.0:1787866184.839443:0:68008:0:(udsp.c:863:lnet_udsp_get_policy()) Get UDSP at idx = 0
00000001:00000200:0.0:1787866184.839445:0:68008:0:(udsp.c:878:lnet_udsp_get_policy()) Found UDSP (00000000fad3d7f9)
00000400:00000010:0.0:1787866184.839448:0:68008:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 32 at 000000004742348e.
00000001:02000400:3.0:1787866184.867588:0:68009:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp --if ens2
00000400:00000010:3.0:1787866184.894042:0:68010:0:(api-ni.c:6877:lnet_net_cmd()) alloc '(conf)': 808 at 000000001578d6a5.
00000400:00000010:3.0:1787866184.894059:0:68010:0:(api-ni.c:6555:lnet_genl_parse_local_ni()) alloc '(tun)': 172 at 00000000ecf7e4c5.
00000400:00000010:3.0:1787866184.894070:0:68010:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 00000000e6f5303d.
00000400:00000010:3.0:1787866184.894074:0:68010:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 552 at 0000000087d0447c.
00000400:00000010:3.0:1787866184.894078:0:68010:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000b8416bbd.
00000400:00000010:3.0:1787866184.894083:0:68010:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000907726e0.
00000400:00000010:3.0:1787866184.894086:0:68010:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a85b7985.
00000400:00000010:3.0:1787866184.894087:0:68010:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006efe2553.
00000400:00000010:3.0:1787866184.894091:0:68010:0:(config.c:397:lnet_ni_add_interface()) alloc '(ni->ni_interface)': 5 at 0000000062a90f6c.
00000400:00000010:3.0:1787866184.894099:0:68010:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000008e0ed7af.
00000400:00000010:3.0:1787866184.894102:0:68010:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000aa17dfbf.
00000800:00000200:0.0:1787866184.899093:0:68010:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=lo ifindex 1 state 0 ns 00000000198b8817
00000800:00000200:0.0:1787866184.899256:0:68010:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=lo ifindex 1 state 0 ns 00000000198b8817
00000800:00000200:0.0:1787866184.899259:0:68010:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=ens2 ifindex 2 state 6 ns 00000000198b8817
00000800:00000200:0.0:1787866184.899261:0:68010:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=ens2 ifindex 2 state 6 ns 00000000198b8817
00000800:00000010:0.0:1787866184.899269:0:68010:0:(socklnd.c:2621:ksocknal_startup()) alloc '(net)': 208 at 0000000050d5d605.
00000800:00000200:0.0:1787866184.899283:0:68010:0:(socklnd_modparams.c:295:ksocklnd_lookup_conns_per_peer()) intf ens2 speed -1
00000400:00000200:1.0:1787866184.899935:0:68010:0:(config.c:1657:lnet_get_link_status_locked()) device idx 1 get_link 2
00000400:00000200:1.0:1787866184.900008:0:68010:0:(acceptor.c:209:lnet_acceptor_add_socket()) Added socket on ens2:988 (IP: 192.168.203.58)
00000400:02000000:1.0:1787866184.900020:0:68010:0:(api-ni.c:2823:lnet_startup_lndni()) Added LNI 192.168.203.58@tcp [8/256/0/180]
00000400:00000010:1.0:1787866184.906291:0:68010:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 57 at 00000000769b4e6d.
00000400:00000010:1.0:1787866184.906303:0:68010:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000007440be0d.
00000400:00000010:1.0:1787866184.906330:0:68010:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e2376687.
00000400:00000010:0.0:1787866184.907273:0:68010:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000084f4344e.
00000400:00000200:0.0:1787866184.907280:0:68010:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000021aec6f6
00000400:00000010:0.0:1787866184.907283:0:68010:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 0000000021aec6f6.
00000400:00000010:0.0:1787866184.907288:0:68010:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 41 at 0000000055425607.
00000400:00000010:0.0:1787866184.907310:0:68010:0:(api-ni.c:6852:lnet_genl_parse_local_ni()) kfreed 'tun': 172 at 00000000ecf7e4c5.
00000400:00000010:0.0:1787866184.907316:0:68010:0:(api-ni.c:6994:lnet_net_cmd()) kfreed 'conf': 808 at 000000001578d6a5.
00000400:00000010:2.0:1787866184.965904:0:68024:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000006ff262b0.
00000400:00000010:2.0:1787866184.965969:0:68024:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000006ff262b0.
00000001:02000400:3.0:1787866185.746243:0:68130:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp1 --if ens2
00000400:00000010:2.0:1787866185.778757:0:68131:0:(api-ni.c:6877:lnet_net_cmd()) alloc '(conf)': 808 at 00000000cd707e0b.
00000400:00000010:2.0:1787866185.778776:0:68131:0:(api-ni.c:6555:lnet_genl_parse_local_ni()) alloc '(tun)': 172 at 0000000095601c89.
00000400:00000010:2.0:1787866185.778785:0:68131:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 000000000e09f060.
00000400:00000010:2.0:1787866185.778790:0:68131:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 552 at 000000007e669fe5.
00000400:00000010:2.0:1787866185.778794:0:68131:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a9abb91b.
00000400:00000010:2.0:1787866185.778799:0:68131:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000aafd4e56.
00000400:00000010:2.0:1787866185.778800:0:68131:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d723100b.
00000400:00000010:2.0:1787866185.778802:0:68131:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007620a0d3.
00000400:00000010:2.0:1787866185.778806:0:68131:0:(config.c:397:lnet_ni_add_interface()) alloc '(ni->ni_interface)': 5 at 000000006ff262b0.
00000800:00000010:2.0:1787866185.778814:0:68131:0:(socklnd.c:2621:ksocknal_startup()) alloc '(net)': 208 at 000000000e3043f0.
00000800:00000200:2.0:1787866185.778825:0:68131:0:(socklnd_modparams.c:295:ksocklnd_lookup_conns_per_peer()) intf ens2 speed -1
00000400:00000200:2.0:1787866185.779237:0:68131:0:(config.c:1657:lnet_get_link_status_locked()) device idx 1 get_link 2
00000400:00000200:2.0:1787866185.779288:0:68131:0:(acceptor.c:209:lnet_acceptor_add_socket()) Added socket on ens2:988 (IP: 192.168.203.58)
00000400:02000000:2.0:1787866185.779307:0:68131:0:(api-ni.c:2823:lnet_startup_lndni()) Added LNI 192.168.203.58@tcp1 [8/256/0/180]
00000400:00000010:2.0:1787866185.785615:0:68131:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 73 at 0000000005408f50.
00000400:00000010:2.0:1787866185.785628:0:68131:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000073d4377f.
00000400:00000010:2.0:1787866185.785654:0:68131:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000053c24a3e.
00000400:00000010:2.0:1787866185.785670:0:68131:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000007440be0d.
00000400:00000200:2.0:1787866185.785673:0:68131:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e2376687
00000400:00000010:2.0:1787866185.785675:0:68131:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000e2376687.
00000400:00000010:2.0:1787866185.785679:0:68131:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 57 at 00000000769b4e6d.
00000400:00000010:2.0:1787866185.786187:0:68131:0:(api-ni.c:6852:lnet_genl_parse_local_ni()) kfreed 'tun': 172 at 0000000095601c89.
00000400:00000010:2.0:1787866185.786193:0:68131:0:(api-ni.c:6994:lnet_net_cmd()) kfreed 'conf': 808 at 00000000cd707e0b.
00000400:00000010:2.0:1787866185.841348:0:68137:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000005b81f753.
00000400:00000010:2.0:1787866185.841411:0:68137:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000005b81f753.
00000400:00000010:0.0:1787866186.584200:0:68255:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000912c74c3.
00000400:00000010:0.0:1787866186.584436:0:68255:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000912c74c3.
00000001:02000400:3.0:1787866186.604341:0:68257:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl discover 192.168.203.58@tcp
00000400:00000010:2.0:1787866186.633934:0:68258:0:(peer.c:230:lnet_peer_alloc()) alloc '(lp)': 360 at 00000000665005b4.
00000400:00000200:2.0:1787866186.634101:0:68258:0:(peer.c:266:lnet_peer_alloc()) 00000000665005b4 nid 192.168.203.58@tcp
00000400:00000010:2.0:1787866186.634107:0:68258:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 0000000060507bb8.
00000400:00000200:2.0:1787866186.634111:0:68258:0:(peer.c:203:lnet_peer_net_alloc()) 0000000060507bb8 net tcp
00000400:00000010:2.0:1787866186.634114:0:68258:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 00000000a7dbe09a.
00000400:00000200:2.0:1787866186.634119:0:68258:0:(peer.c:184:lnet_peer_ni_alloc()) 00000000a7dbe09a nid 192.168.203.58@tcp
00000400:00000200:2.0:1787866186.634127:0:68258:0:(peer.c:1804:lnet_peer_attach_peer_ni()) peer 192.168.203.58@tcp NID 192.168.203.58@tcp flags 0x0
00000400:00000200:2.0:1787866186.634132:0:68258:0:(peer.c:2183:lnet_peer_ni_traffic_add()) peer 192.168.203.58@tcp: 0
00000400:00000200:2.0:1787866186.634159:0:68258:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.203.58@tcp: 0
00000400:00000200:2.0:1787866186.634162:0:68258:0:(peer.c:2935:lnet_discover_peer_locked()) Discovery attempt # 1
00000400:00000200:1.0:1787866186.634211:0:68007:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0
00000400:00000200:1.0:1787866186.634237:0:68007:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000665005b4) state 0x40
00000400:00000010:1.0:1787866186.634249:0:68007:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000a9f9657c.
00000400:00000010:1.0:1787866186.634256:0:68007:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000c7efde77.
00000400:00000010:1.0:1787866186.634303:0:68007:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000925783b6.
00000400:00000200:1.0:1787866186.634309:0:68007:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.203.58@tcp
00000400:00000200:1.0:1787866186.634323:0:68007:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR:  192.168.203.58@tcp local destination
00000400:00000200:1.0:1787866186.634330:0:68007:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.58@tcp
00000400:00000200:1.0:1787866186.634333:0:68007:0:(lib-move.c:1874:lnet_set_non_mr_pref_nid()) Setting preferred local NID 192.168.203.58@tcp on NMR peer 192.168.203.58@tcp
00000400:00000200:1.0:1787866186.634336:0:68007:0:(peer.c:1171:lnet_peer_ni_set_non_mr_pref_nid()) peer 192.168.203.58@tcp nid 192.168.203.58@tcp: 0
00000400:00000200:1.0:1787866186.634341:0:68007:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [1:1:256:4294967295] 192.168.203.58@tcp LPNI seq info [1:1:8:4294967295]
00000400:00000200:1.0:1787866186.634348:0:68007:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:1.0:1787866186.634354:0:68007:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:<?>) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : GET try# 0
00000800:00000200:1.0:1787866186.634360:0:68007:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.203.58@tcp
00000800:00000010:1.0:1787866186.634366:0:68007:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000002268f524.
00000800:00000010:1.0:1787866186.634376:0:68007:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 00000000d15e6937.
00000800:00000010:1.0:1787866186.634380:0:68007:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 0000000040eb0d5e.
00000800:00000200:1.0:1787866186.634384:0:68007:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000d15e6937] -> 12345-192.168.203.58@tcp (2)
00000800:00000200:1.1F:1787866186.634388:0:68007:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000d15e6937] -> 12345-192.168.203.58@tcp (2)
00000400:00000200:1.0:1787866186.634436:0:68007:0:(peer.c:4022:lnet_peer_send_ping()) peer 192.168.203.58@tcp
00000400:00000200:1.0:1787866186.634439:0:68007:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000665005b4) state 0x240 rc 0
00000400:00000200:1.0:1787866186.634978:0:68019:0:(acceptor.c:578:lnet_acceptor()) Accepted connection on ens2:988
00000400:00000200:1.0:1787866186.635003:0:68019:0:(acceptor.c:502:lnet_accept()) Accept 192.168.203.58@tcp from 255.3.0.2
00000800:00000010:1.0:1787866186.635006:0:68019:0:(socklnd.c:737:ksocknal_accept()) alloc '(cr)': 32 at 00000000ac98b577.
00000800:00000010:2.0:1787866186.635123:0:68012:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 00000000f783d8b3.
00000800:00000010:2.0:1787866186.635136:0:68012:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 0000000095601c89.
00000800:00000010:2.0:1787866186.635145:0:68012:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000905a879b.
00000800:00000010:2.0:1787866186.635199:0:68012:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000905a879b.
00000800:00000010:0.0:1787866186.640166:0:68011:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 0000000087eabc9c.
00000800:00000010:0.0:1787866186.640179:0:68011:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 0000000021faa4ca.
00000800:00000010:0.0:1787866186.640211:0:68011:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 00000000ecf7e4c5.
00000800:00000200:0.1F:1787866186.640222:0:68011:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000d15e6937] -> 12345-192.168.203.58@tcp (2)
00000800:00000200:0.1:1787866186.640933:0:68011:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.203.58@tcp 00000000ecf7e4c5 deleted
00000800:00000010:0.1:1787866186.640935:0:68011:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 00000000ecf7e4c5.
00000800:00000200:0.1:1787866186.640941:0:68011:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1
00000800:00000200:0.1:1787866186.640947:0:68011:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000200:0.1:1787866186.640952:0:68011:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002268f524 type 2, nob 96 niov 1 nkiov 0
00000800:00000200:0.0:1787866186.640957:0:68011:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.203.58@tcp p 3.x 192.168.203.58 -> 192.168.203.58:1023 incarnation:1787866184899271558 sched[0]
00000800:00000010:1.0:1787866186.644820:0:68011:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 000000004da2bfa9.
00000800:00000010:1.0:1787866186.644972:0:68011:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 000000004da2bfa9.
00000800:00000010:1.0:1787866186.644984:0:68011:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 0000000021faa4ca.
00000800:00000010:1.0:1787866186.645021:0:68011:0:(socklnd_cb.c:2179:ksocknal_connd()) kfreed 'cr': 32 at 00000000ac98b577.
00000800:00000200:2.1F:1787866186.645150:0:68012:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1
00000800:00000200:2.0:1787866186.645184:0:68012:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.203.58@tcp p 3.x 192.168.203.58 -> 192.168.203.58:988 incarnation:1787866184899271558 sched[0]
00000800:00000010:2.0:1787866186.645189:0:68012:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 0000000095601c89.
00000400:00000200:2.0:1787866186.645318:0:68012:0:(lib-socket.c:438:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.203.58:988
00000800:00000200:0.0:1787866186.645330:0:68018:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:0.0:1787866186.645336:0:68018:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000002268f524.
00000400:00000200:0.0:1787866186.645343:0:68018:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1787866186.645353:0:68018:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: GET: OK
00000400:00000200:0.0:1787866186.645361:0:68018:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 5
00000400:00000200:0.0:1787866186.645365:0:68018:0:(peer.c:3241:lnet_discovery_event_send()) Ping Send to 192.168.203.58@tcp: 0
00000400:00000200:0.0:1787866186.645415:0:68018:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : GET - for me
00000400:00000200:0.0:1787866186.645447:0:68018:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.58@tcp of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:0.0:1787866186.645455:0:68018:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.203.58@tcp of length 64/272 into md 0xd [1] + 0
00000400:00000200:0.0:1787866186.645480:0:68018:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.203.58@tcp to NMR:  192.168.203.58@tcp local destination
00000400:00000200:0.0:1787866186.645486:0:68018:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [2:2:256:4294967295] 192.168.203.58@tcp LPNI seq info [2:2:8:4294967295]
00000400:00000200:0.0:1787866186.645501:0:68018:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) 192.168.203.58@tcp : REPLY try# 0
00000800:00000200:0.0:1787866186.645507:0:68018:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.203.58@tcp
00000800:00000010:0.0:1787866186.645514:0:68018:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000ada1255e.
00000800:00000200:0.0:1787866186.645520:0:68018:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000d15e6937] -> 12345-192.168.203.58@tcp (4)
00000800:00000200:0.0:1787866186.645526:0:68018:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000200:0.0:1787866186.645530:0:68018:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ada1255e type 3, nob 160 niov 1 nkiov 1
00000800:00000200:0.0:1787866186.645600:0:68018:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:0.0:1787866186.645603:0:68018:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000ada1255e.
00000400:00000200:0.0:1787866186.645608:0:68018:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1787866186.645614:0:68018:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: REPLY: OK
00000400:00000200:0.0:1787866186.645638:0:68018:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : REPLY - for me
00000800:00000010:2.0:1787866186.645644:0:68012:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 00000000b1b59a22.
00000800:00000010:2.0:1787866186.645654:0:68012:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 0000000095601c89.
00000400:00000200:0.0:1787866186.645655:0:68018:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.203.58@tcp: Reply from 12345-192.168.203.58@tcp of length 64/64 into md 0x11
00000800:00000010:2.0:1787866186.645665:0:68012:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000905a879b.
00000400:00000200:0.0:1787866186.645670:0:68018:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1787866186.645676:0:68018:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: REPLY: OK
00000400:00000200:0.0:1787866186.645681:0:68018:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c7efde77
00000400:00000010:0.0:1787866186.645684:0:68018:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000c7efde77.
00000400:00000200:0.0:1787866186.645688:0:68018:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 3
00000400:00000200:0.0:1787866186.645692:0:68018:0:(peer.c:3119:lnet_discovery_event_reply()) Peer 192.168.203.58@tcp has discovery enabled
00000400:00000200:0.0:1787866186.645705:0:68018:0:(peer.c:3145:lnet_discovery_event_reply()) peer 192.168.203.58@tcp(00000000665005b4) is MR capable
00000800:00000010:2.0:1787866186.645707:0:68012:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000905a879b.
00000400:00000200:0.0:1787866186.645715:0:68018:0:(peer.c:1196:lnet_peer_ni_clr_non_mr_pref_nid()) peer 192.168.203.58@tcp: 0
00000400:00000200:0.0:1787866186.645722:0:68018:0:(peer.c:3200:lnet_discovery_event_reply()) peer 192.168.203.58@tcp data present 3. state = 0x241
00000400:00000200:0.0:1787866186.645726:0:68018:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.203.58@tcp: -114
00000400:00000200:2.0:1787866186.645766:0:68007:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0
00000400:00000200:2.0:1787866186.645774:0:68007:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000665005b4) state 0xc1
00000400:00000010:2.0:1787866186.645783:0:68007:0:(peer.c:3474:lnet_peer_merge_data()) alloc '(curnis)': 60 at 00000000905a879b.
00000400:00000010:2.0:1787866186.645787:0:68007:0:(peer.c:3475:lnet_peer_merge_data()) alloc '(addnis)': 72 at 00000000c980c965.
00000400:00000010:2.0:1787866186.645789:0:68007:0:(peer.c:3476:lnet_peer_merge_data()) alloc '(delnis)': 60 at 00000000d014d1b7.
00000400:00000010:2.0:1787866186.645800:0:68007:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 00000000aee8cefc.
00000400:00000200:2.0:1787866186.645805:0:68007:0:(peer.c:184:lnet_peer_ni_alloc()) 00000000aee8cefc nid 192.168.203.58@tcp1
00000400:00000010:2.0:1787866186.645809:0:68007:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 000000001b28f015.
00000400:00000200:2.0:1787866186.645811:0:68007:0:(peer.c:203:lnet_peer_net_alloc()) 000000001b28f015 net tcp1
00000400:00000200:2.0:1787866186.645818:0:68007:0:(peer.c:1804:lnet_peer_attach_peer_ni()) peer 192.168.203.58@tcp NID 192.168.203.58@tcp1 flags 0x11
00000400:00000010:2.0:1787866186.645823:0:68007:0:(peer.c:3623:lnet_peer_merge_data()) kfreed 'curnis': 60 at 00000000905a879b.
00000400:00000010:2.0:1787866186.645825:0:68007:0:(peer.c:3624:lnet_peer_merge_data()) kfreed 'addnis': 72 at 00000000c980c965.
00000400:00000010:2.0:1787866186.645827:0:68007:0:(peer.c:3625:lnet_peer_merge_data()) kfreed 'delnis': 60 at 00000000d014d1b7.
00000400:00000010:2.0:1787866186.645853:0:68007:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000a9f9657c.
00000400:00000200:2.0:1787866186.645859:0:68007:0:(peer.c:3628:lnet_peer_merge_data()) peer 192.168.203.58@tcp (00000000665005b4): 0
00000400:00000200:2.0:1787866186.645862:0:68007:0:(peer.c:3940:lnet_peer_data_present()) peer 192.168.203.58@tcp(00000000665005b4): 0. state = 0x141
00000400:00000200:2.0:1787866186.645865:0:68007:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000665005b4) state 0x141 rc 1
00000400:00000200:2.0:1787866186.645868:0:68007:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000665005b4) state 0x141
00000400:00000010:2.0:1787866186.645875:0:68007:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001305b46c.
00000400:00000010:2.0:1787866186.645902:0:68007:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000002a00e456.
00000400:00000200:2.0:1787866186.645913:0:68007:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.58@tcp
00000400:00000200:2.0:1787866186.645925:0:68007:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.203.58@tcp to MR:  192.168.203.58@tcp local destination
00000400:00000200:2.0:1787866186.645931:0:68007:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [3:3:256:4294967295] 192.168.203.58@tcp LPNI seq info [3:3:8:4294967295]
00000400:00000200:2.0:1787866186.645936:0:68007:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:2.0:1787866186.645942:0:68007:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787866186.645947:0:68007:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.203.58@tcp
00000800:00000010:2.0:1787866186.645951:0:68007:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000013eaff37.
00000400:00000200:1.0:1787866186.646097:0:68019:0:(acceptor.c:578:lnet_acceptor()) Accepted connection on ens2:988
00000400:00000200:1.0:1787866186.646114:0:68019:0:(acceptor.c:502:lnet_accept()) Accept 192.168.203.58@tcp from 254.3.0.2
00000800:00000010:1.0:1787866186.646117:0:68019:0:(socklnd.c:737:ksocknal_accept()) alloc '(cr)': 32 at 00000000ac98b577.
00000800:00000200:2.0:1787866186.646347:0:68007:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000d15e6937] -> 12345-192.168.203.58@tcp (4)
00000800:00000200:2.0:1787866186.646354:0:68007:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000200:2.0:1787866186.646358:0:68007:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000013eaff37 type 1, nob 160 niov 1 nkiov 1
00000400:00000200:2.0:1787866186.646380:0:68007:0:(peer.c:4164:lnet_peer_send_push()) peer 192.168.203.58@tcp
00000400:00000200:2.0:1787866186.646383:0:68007:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000665005b4) state 0x541 rc 0
00000800:00000200:3.0:1787866186.646523:0:68016:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1787866186.646528:0:68016:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000013eaff37.
00000400:00000200:3.0:1787866186.646533:0:68016:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1787866186.646548:0:68016:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: PUT: OK
00000400:00000200:3.0:1787866186.646554:0:68016:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 5
00000400:00000200:3.0:1787866186.646557:0:68016:0:(peer.c:3241:lnet_discovery_event_send()) Push Send to 192.168.203.58@tcp: 0
00000400:00000200:3.0:1787866186.646584:0:68016:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : PUT - for me
00000800:00000010:0.0:1787866186.646590:0:68014:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 00000000bba29d83.
00000800:00000010:0.0:1787866186.646598:0:68014:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 00000000ecf7e4c5.
00000400:00000200:3.0:1787866186.646617:0:68016:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.58@tcp of length 64 into portal 0 MB=0x8000000000000000
00000400:00000200:3.0:1787866186.646627:0:68016:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 0 from 12345-192.168.203.58@tcp of length 64/64 into md 0x5 [1] + 0
00000800:00000010:0.0:1787866186.646628:0:68014:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 00000000dbcc7172.
00000400:00000010:3.0:1787866186.646637:0:68016:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b3af47a7.
00000400:00000200:3.0:1787866186.646649:0:68016:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000a3edd4bd
00000800:00000200:0.1:1787866186.646649:0:68014:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000d15e6937] -> 12345-192.168.203.58@tcp (4)
00000800:00000200:0.1:1787866186.646660:0:68014:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.203.58@tcp 00000000dbcc7172 deleted
00000800:00000010:0.1:1787866186.646663:0:68014:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 00000000dbcc7172.
00000800:00000200:0.1:1787866186.646770:0:68014:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 3, ksnr_connected a ksnr_max_conns 1
00000800:00000200:0.0:1787866186.646776:0:68014:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.203.58@tcp p 3.x 192.168.203.58 -> 192.168.203.58:1022 incarnation:1787866184899271558 sched[0]
00000800:00000010:0.0:1787866186.646783:0:68014:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 0000000014122af9.
00000400:00000200:3.0:1787866186.646814:0:68016:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1787866186.646823:0:68016:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: PUT: OK
00000400:00000200:3.0:1787866186.646828:0:68016:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a3edd4bd
00000400:00000010:3.0:1787866186.646875:0:68016:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000a3edd4bd.
00000400:00000200:3.0:1787866186.646884:0:68016:0:(api-ni.c:2393:lnet_push_target_event_handler()) type 2 status 0 unlinked 1
00000400:00000200:3.0:1787866186.646894:0:68016:0:(peer.c:2778:lnet_peer_push_event()) peer 192.168.203.58@tcp(00000000665005b4) is MR
00000400:00000010:3.0:1787866186.646902:0:68016:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 73 at 00000000938cb033.
00000400:00000200:3.0:1787866186.646906:0:68016:0:(peer.c:2852:lnet_peer_push_event()) Received Push 192.168.203.58@tcp 3
00000400:00000200:3.0:1787866186.646913:0:68016:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.203.58@tcp: -114
00000800:00000010:0.0:1787866186.646926:0:68014:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 0000000014122af9.
00000400:00000200:3.0:1787866186.646933:0:68016:0:(lib-md.c:96:lnet_cpt_of_md()) Couldn't resolve first page of md 0000000000000000 with offset 0
00000800:00000010:0.0:1787866186.646935:0:68014:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 00000000ecf7e4c5.
00000400:00000200:3.0:1787866186.646947:0:68016:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.203.58@tcp to MR:  192.168.203.58@tcp local destination
00000400:00000200:3.0:1787866186.646964:0:68016:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [4:4:256:4294967295] 192.168.203.58@tcp LPNI seq info [4:4:8:4294967295]
00000400:00000200:3.0:1787866186.646974:0:68016:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) 192.168.203.58@tcp : ACK try# 0
00000800:00000200:3.0:1787866186.646979:0:68016:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 1 frags to 12345-192.168.203.58@tcp
00000800:00000200:2.1:1787866186.647061:0:68012:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 2, ksnr_connected e ksnr_max_conns 1
00000800:00000200:2.0:1787866186.647069:0:68012:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.203.58@tcp p 3.x 192.168.203.58 -> 192.168.203.58:988 incarnation:1787866184899271558 sched[0]
00000800:00000010:2.0:1787866186.647073:0:68012:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 0000000095601c89.
00000800:00000010:3.0:1787866186.647120:0:68016:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000025ecaeeb.
00000800:00000200:3.0:1787866186.647125:0:68016:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000d15e6937] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:3.0:1787866186.647129:0:68016:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000200:3.0:1787866186.647131:0:68016:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000025ecaeeb type 0, nob 96 niov 1 nkiov 0
00000800:00000200:2.0:1787866186.648032:0:68017:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1787866186.648039:0:68017:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000025ecaeeb.
00000400:00000200:2.0:1787866186.648044:0:68017:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1787866186.648051:0:68017:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: ACK: OK
00000400:00000200:2.0:1787866186.648071:0:68017:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : ACK - for me
00000800:00000010:0.0:1787866186.648083:0:68014:0:(socklnd_cb.c:2179:ksocknal_connd()) kfreed 'cr': 32 at 00000000ac98b577.
00000400:00000200:2.0:1787866186.648092:0:68017:0:(lib-move.c:4539:lnet_parse_ack()) 192.168.203.58@tcp: ACK from 12345-192.168.203.58@tcp into md 0x15
00000400:00000200:2.0:1787866186.648098:0:68017:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1787866186.648102:0:68017:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: ACK: OK
00000400:00000200:2.0:1787866186.648108:0:68017:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001305b46c
00000400:00000010:2.0:1787866186.648111:0:68017:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000001305b46c.
00000400:00000200:2.0:1787866186.648116:0:68017:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 4
00000400:00000200:2.0:1787866186.648121:0:68017:0:(peer.c:3007:lnet_discovery_event_ack()) peer 192.168.203.58@tcp ev->status 0
00000400:00000200:2.0:1787866186.648125:0:68017:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.203.58@tcp: -114
00000400:00000200:1.0:1787866186.648531:0:68007:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0
00000400:00000010:1.0:1787866186.648543:0:68007:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000352a51e8.
00000400:00000010:1.0:1787866186.648551:0:68007:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b3cf5b6b.
00000400:00000200:1.0:1787866186.648557:0:68007:0:(api-ni.c:2383:lnet_push_target_post()) posted push target 00000000ffa05391
00000400:00000200:1.0:1787866186.648565:0:68007:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000665005b4) state 0xc1
00000400:00000010:1.0:1787866186.648644:0:68007:0:(peer.c:3474:lnet_peer_merge_data()) alloc '(curnis)': 60 at 000000004da2bfa9.
00000400:00000010:1.0:1787866186.648648:0:68007:0:(peer.c:3475:lnet_peer_merge_data()) alloc '(addnis)': 72 at 00000000dd3d2298.
00000400:00000010:1.0:1787866186.648650:0:68007:0:(peer.c:3476:lnet_peer_merge_data()) alloc '(delnis)': 60 at 000000000552eca6.
00000400:00000010:1.0:1787866186.648657:0:68007:0:(peer.c:3623:lnet_peer_merge_data()) kfreed 'curnis': 60 at 000000004da2bfa9.
00000400:00000010:1.0:1787866186.648660:0:68007:0:(peer.c:3624:lnet_peer_merge_data()) kfreed 'addnis': 72 at 00000000dd3d2298.
00000400:00000010:1.0:1787866186.648662:0:68007:0:(peer.c:3625:lnet_peer_merge_data()) kfreed 'delnis': 60 at 000000000552eca6.
00000400:00000010:1.0:1787866186.648665:0:68007:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 73 at 00000000938cb033.
00000400:00000200:1.0:1787866186.648669:0:68007:0:(peer.c:3628:lnet_peer_merge_data()) peer 192.168.203.58@tcp (00000000665005b4): 0
00000400:00000200:1.0:1787866186.648672:0:68007:0:(peer.c:3940:lnet_peer_data_present()) peer 192.168.203.58@tcp(00000000665005b4): 0. state = 0x141
00000400:00000200:1.0:1787866186.648675:0:68007:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000665005b4) state 0x141 rc 1
00000400:00000200:1.0:1787866186.648678:0:68007:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000665005b4) state 0x141
00000400:00000200:1.0:1787866186.648681:0:68007:0:(peer.c:4082:lnet_peer_discovered()) peer 192.168.203.58@tcp
00000400:00000200:1.0:1787866186.648683:0:68007:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000665005b4) state 0x111 rc 0
00000400:00000200:1.0:1787866186.648686:0:68007:0:(peer.c:2629:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 192.168.203.58@tcp
00000400:00000200:2.0:1787866186.648798:0:68258:0:(peer.c:2984:lnet_discover_peer_locked()) peer 192.168.203.58@tcp NID 192.168.203.58@tcp: 0. discovery complete
00000400:00000010:2.0:1787866186.674774:0:68260:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 12 at 00000000701205b7.
00000400:00000010:2.0:1787866186.674789:0:68260:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 12 at 00000000701205b7.
00000001:02000400:1.0:1787866186.713553:0:68262:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 10
00000001:02000400:2.0:1787866186.775524:0:68265:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp -d *@tcp -m GET -r 1 -e local_error
00000400:00000200:3.0:1787866186.822535:0:68266:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866186.822545:0:68266:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866186.822556:0:68266:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp
00000400:00000200:3.0:1787866186.822558:0:68266:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866186.822561:0:68266:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp
00000400:00000200:3.0:1787866186.822569:0:68266:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866186.822572:0:68266:0:(api-ni.c:3207:nla_strnid()) <?> -> <?>
00000400:00000200:3.0:1787866186.822573:0:68266:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866186.822574:0:68266:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866186.822576:0:68266:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866186.822577:0:68266:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866186.822579:0:68266:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866186.822580:0:68266:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866186.822582:0:68266:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000010:3.0:1787866186.822587:0:68266:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 0000000044c03cfa.
00000400:00000200:3.0:1787866186.822595:0:68266:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp, dst 255.255.255.255@tcp, rate 1, interval 0
00000001:02000400:0.0:1787866186.866620:0:68267:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp1 -d *@tcp1 -m GET -r 1 -e local_error
00000400:00000200:2.0:1787866186.904641:0:68268:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866186.904649:0:68268:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866186.904660:0:68268:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1
00000400:00000200:2.0:1787866186.904662:0:68268:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866186.904665:0:68268:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1
00000400:00000200:2.0:1787866186.904666:0:68268:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866186.904669:0:68268:0:(api-ni.c:3207:nla_strnid()) <?> -> <?>
00000400:00000200:2.0:1787866186.904670:0:68268:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866186.904671:0:68268:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866186.904673:0:68268:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866186.904674:0:68268:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866186.904676:0:68268:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866186.904677:0:68268:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866186.904679:0:68268:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000010:2.0:1787866186.904683:0:68268:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 00000000b7b9bb79.
00000400:00000200:2.0:1787866186.904689:0:68268:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp1, dst 255.255.255.255@tcp1, rate 1, interval 0
00000001:02000400:1.0:1787866186.920666:0:68269:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net_drop add -s *@tcp -d *@tcp -r 1
00000400:00000200:1.0:1787866186.942740:0:68270:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866186.942750:0:68270:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866186.942761:0:68270:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp
00000400:00000200:1.0:1787866186.942765:0:68270:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866186.942767:0:68270:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp
00000400:00000200:1.0:1787866186.942768:0:68270:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866186.942770:0:68270:0:(api-ni.c:3207:nla_strnid()) <?> -> <?>
00000400:00000200:1.0:1787866186.942771:0:68270:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866186.942780:0:68270:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866186.942781:0:68270:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866186.942782:0:68270:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866186.942784:0:68270:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866186.942785:0:68270:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866186.942787:0:68270:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000010:1.0:1787866186.942791:0:68270:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 0000000061201bde.
00000400:00000200:1.0:1787866186.942799:0:68270:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp, dst 255.255.255.255@tcp, rate 1, interval 0
00000001:02000400:0.0:1787866186.981123:0:68271:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp1 -d *@tcp1 -r 1
00000400:00000200:3.0:1787866187.011979:0:68272:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866187.011988:0:68272:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866187.012001:0:68272:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1
00000400:00000200:3.0:1787866187.012002:0:68272:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866187.012005:0:68272:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1
00000400:00000200:3.0:1787866187.012007:0:68272:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866187.012009:0:68272:0:(api-ni.c:3207:nla_strnid()) <?> -> <?>
00000400:00000200:3.0:1787866187.012010:0:68272:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866187.012012:0:68272:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866187.012013:0:68272:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866187.012014:0:68272:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866187.012016:0:68272:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866187.012017:0:68272:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866187.012019:0:68272:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000010:3.0:1787866187.012022:0:68272:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 00000000265e7ce9.
00000400:00000200:3.0:1787866187.012029:0:68272:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp1, dst 255.255.255.255@tcp1, rate 1, interval 0
00000001:02000400:3.0:1787866187.021748:0:68273:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net set --health 0 --nid 192.168.203.58@tcp
00000400:00000010:2.0:1787866187.041611:0:68274:0:(api-ni.c:6877:lnet_net_cmd()) alloc '(conf)': 808 at 000000007155bf16.
00000400:00000010:2.0:1787866187.041630:0:68274:0:(api-ni.c:6555:lnet_genl_parse_local_ni()) alloc '(tun)': 172 at 0000000095601c89.
00000400:00000200:2.0:1787866187.041634:0:68274:0:(api-ni.c:6621:lnet_genl_parse_local_ni()) Requested NID 192.168.203.58@tcp
00000400:00020000:2.0:1787866187.041648:0:68274:0:(api-ni.c:4473:lnet_ni_set_healthv()) manually adding local NI 192.168.203.58@tcp to recovery
00000400:00000200:2.0:1787866187.055354:0:68274:0:(lib-lnet.h:200:lnet_ni_set_status_locked()) ni 192.168.203.58@tcp status changed from 0x15aac0de to 0xdeadface
00000400:00000010:2.0:1787866187.055362:0:68274:0:(api-ni.c:6852:lnet_genl_parse_local_ni()) kfreed 'tun': 172 at 0000000095601c89.
00000400:00000010:2.0:1787866187.055370:0:68274:0:(api-ni.c:6994:lnet_net_cmd()) kfreed 'conf': 808 at 000000007155bf16.
00000400:00000010:2.0:1787866187.119179:0:68280:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000005b81f753.
00000400:00000010:2.0:1787866187.120754:0:68280:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000005b81f753.
00000400:00000200:3.0:1787866187.923027:0:68006:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.203.58@tcp
00000400:00000010:3.0:1787866187.923041:0:68006:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000074a66375.
00000400:00000010:3.0:1787866187.923052:0:68006:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000002a3fb895.
00000400:00000010:3.0:1787866187.923060:0:68006:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a3edd4bd.
00000400:00000010:3.0:1787866187.923071:0:68006:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000006f559dd9.
00000400:00000200:3.0:1787866187.923075:0:68006:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.203.58@tcp
00000400:00000200:3.0:1787866187.923094:0:68006:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR:  192.168.203.58@tcp local destination
00000400:00000200:3.0:1787866187.923102:0:68006:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.58@tcp
00000400:00000200:3.0:1787866187.923106:0:68006:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [5:5:256:4294967295] 192.168.203.58@tcp LPNI seq info [5:5:8:4294967295]
00000400:00000200:3.0:1787866187.923114:0:68006:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:3.0:1787866187.923138:0:68006:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:<?>) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : GET try# 0
00000800:00000200:3.0:1787866187.923144:0:68006:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.203.58@tcp
00000800:00000010:3.0:1787866187.923149:0:68006:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000025ecaeeb.
00000800:00000200:3.0:1787866187.923158:0:68006:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000d15e6937] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:3.0:1787866187.923167:0:68006:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000200:3.0:1787866187.923172:0:68006:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000025ecaeeb type 2, nob 96 niov 1 nkiov 0
00000400:00000200:3.0:1787866187.923198:0:68006:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.203.58@tcp added to recovery queue. ping count: 1 next ping: 1399 health :0
00000400:00000010:3.0:1787866187.923206:0:68006:0:(lib-move.c:3814:lnet_print_recovery_list()) alloc '(buf)': 65 at 00000000938cb033.
00000400:02000040:3.0:1787866187.923210:0:68006:0:(lib-move.c:3830:lnet_print_recovery_list()) 1 local NIs in recovery (showing 1): 192.168.203.58@tcp
00000400:00000200:0.0:1787866187.923298:0:68018:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 1
00000400:00000200:0.0:1787866187.923308:0:68018:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.203.58@tcp(192.168.203.58@tcp)->dst 192.168.203.58@tcp: GET simulate health error: LOCAL_ERROR
00000800:02000400:0.0:1787866187.923312:0:68018:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.203.58: rc = -22
00000400:00000010:3.0:1787866187.926121:0:68006:0:(lib-move.c:3832:lnet_print_recovery_list()) kfreed 'buf': 65 at 00000000938cb033.
00000800:00000200:0.0:1787866187.932674:0:68018:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [00000000f783d8b3] Error -22 on write to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000010:0.0:1787866187.932689:0:68018:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000025ecaeeb.
00000400:00000200:0.0:1787866187.932701:0:68018:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 5
00000400:00000200:0.0:1787866187.932708:0:68018:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: GET: LOCAL_ERROR
00000400:00000200:0.0:1787866187.932721:0:68018:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5
00000400:00000200:0.0:1787866187.932725:0:68018:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.203.58@tcp recovery message sent unsuccessfully:-5
00000400:00000200:0.0:1787866187.932728:0:68018:0:(lib-move.c:4071:lnet_handle_recovery_reply()) local NI (192.168.203.58@tcp) recovery failed with -5
00000400:00000010:1.0:1787866188.248194:0:68290:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000001f6718ce.
00000400:00000010:1.0:1787866188.248322:0:68290:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000001f6718ce.
00000400:00000200:3.0:1787866188.947012:0:68006:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a3edd4bd
00000400:00000010:3.0:1787866188.947026:0:68006:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000a3edd4bd.
00000400:00000200:3.0:1787866188.947036:0:68006:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0
00000400:00000200:3.0:1787866188.947046:0:68006:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.203.58@tcp recovery ping unlinked
00000400:00000010:3.0:1787866188.947051:0:68006:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000074a66375.
00000400:00000010:3.0:1787866188.947057:0:68006:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000002a3fb895.
00000400:00000010:1.0:1787866189.398294:0:68300:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000001f6718ce.
00000400:00000010:1.0:1787866189.398420:0:68300:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000001f6718ce.
00000400:00000200:3.0:1787866189.972008:0:68006:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.203.58@tcp
00000400:00000010:3.0:1787866189.972023:0:68006:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000000874e15d.
00000400:00000010:3.0:1787866189.972035:0:68006:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000099ab2b98.
00000400:00000010:3.0:1787866189.972043:0:68006:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a3edd4bd.
00000400:00000010:3.0:1787866189.972053:0:68006:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000d1c331d2.
00000400:00000200:3.0:1787866189.972059:0:68006:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.203.58@tcp
00000400:00000200:3.0:1787866189.972077:0:68006:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR:  192.168.203.58@tcp local destination
00000400:00000200:3.0:1787866189.972085:0:68006:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.58@tcp
00000400:00000200:3.0:1787866189.972090:0:68006:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [6:6:256:4294967295] 192.168.203.58@tcp LPNI seq info [6:6:8:4294967295]
00000400:00000200:3.0:1787866189.972098:0:68006:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:3.0:1787866189.972106:0:68006:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:<?>) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : GET try# 0
00000800:00000200:3.0:1787866189.972112:0:68006:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.203.58@tcp
00000800:00000010:3.0:1787866189.972118:0:68006:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000040999296.
00000800:00000200:3.0:1787866189.972125:0:68006:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000d15e6937] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:3.0:1787866189.972153:0:68006:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000200:3.0:1787866189.972158:0:68006:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000040999296 type 2, nob 96 niov 1 nkiov 0
00000400:00000200:3.0:1787866189.972187:0:68006:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.203.58@tcp added to recovery queue. ping count: 2 next ping: 1403 health :0
00000400:00000200:2.0:1787866189.974130:0:68017:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 2
00000400:00000200:2.0:1787866189.974142:0:68017:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.203.58@tcp(192.168.203.58@tcp)->dst 192.168.203.58@tcp: GET simulate health error: LOCAL_ERROR
00000800:02000400:2.0:1787866189.974145:0:68017:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.203.58: rc = -22
00000800:00000200:2.0:1787866189.982464:0:68017:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [0000000087eabc9c] Error -22 on write to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000010:2.0:1787866189.982480:0:68017:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000040999296.
00000400:00000200:2.0:1787866189.982490:0:68017:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 5
00000400:00000200:2.0:1787866189.982497:0:68017:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: GET: LOCAL_ERROR
00000400:00000200:2.0:1787866189.982509:0:68017:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5
00000400:00000200:2.0:1787866189.982536:0:68017:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.203.58@tcp recovery message sent unsuccessfully:-5
00000400:00000200:2.0:1787866189.982540:0:68017:0:(lib-move.c:4071:lnet_handle_recovery_reply()) local NI (192.168.203.58@tcp) recovery failed with -5
00000400:00000010:3.0:1787866190.518531:0:68310:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000005f02433f.
00000400:00000010:3.0:1787866190.518661:0:68310:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000005f02433f.
00000400:00000010:2.0:1787866190.573913:0:68314:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000005b81f753.
00000400:00000010:2.0:1787866190.574055:0:68314:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000005b81f753.
00000400:00000200:2.0:1787866190.642808:0:68321:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 0
00000400:00000010:2.0:1787866190.642820:0:68321:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000b8140383.
00000400:00000200:2.0:1787866190.642856:0:68321:0:(api-ni.c:9567:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:2.0:1787866190.642876:0:68321:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000eb5843d2.
00000400:00000200:2.0:1787866190.642885:0:68321:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:2.0:1787866190.642890:0:68321:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000eb5843d2.
00000400:00000010:2.0:1787866190.642902:0:68321:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000b8140383.
00000001:02000400:2.0:1787866190.685040:0:68323:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -l
00000400:00000200:3.0:1787866190.737168:0:68324:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 0
00000400:00000010:3.0:1787866190.737183:0:68324:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000002df42b09.
00000400:00000200:3.0:1787866190.737196:0:68324:0:(api-ni.c:9567:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:3.0:1787866190.737218:0:68324:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 000000008647e19c.
00000400:00000200:3.0:1787866190.737227:0:68324:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:3.0:1787866190.737233:0:68324:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 000000008647e19c.
00000400:00000010:3.0:1787866190.737242:0:68324:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000002df42b09.
00000400:00000010:1.0:1787866190.806637:0:68330:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000001f6718ce.
00000400:00000010:1.0:1787866190.806826:0:68330:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000001f6718ce.
00000400:00000200:3.0:1787866190.996181:0:68006:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a3edd4bd
00000400:00000010:3.0:1787866190.996196:0:68006:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000a3edd4bd.
00000400:00000200:3.0:1787866190.996207:0:68006:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0
00000400:00000200:3.0:1787866190.996218:0:68006:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.203.58@tcp recovery ping unlinked
00000400:00000010:3.0:1787866190.996224:0:68006:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000000874e15d.
00000400:00000010:3.0:1787866190.996230:0:68006:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000099ab2b98.
00000400:00000010:2.0:1787866191.940356:0:68341:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000005b81f753.
00000400:00000010:2.0:1787866191.940500:0:68341:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000005b81f753.
00000400:00000010:2.0:1787866193.034964:0:68351:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000005b81f753.
00000400:00000010:2.0:1787866193.036694:0:68351:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000005b81f753.
00000400:00000200:3.0:1787866194.066974:0:68006:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.203.58@tcp
00000400:00000010:3.0:1787866194.066984:0:68006:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000c58a0ec5.
00000400:00000010:3.0:1787866194.066994:0:68006:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000099ab2b98.
00000400:00000010:3.0:1787866194.067000:0:68006:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a3edd4bd.
00000400:00000010:3.0:1787866194.067008:0:68006:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000fab4034e.
00000400:00000200:3.0:1787866194.067012:0:68006:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.203.58@tcp
00000400:00000200:3.0:1787866194.067026:0:68006:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR:  192.168.203.58@tcp local destination
00000400:00000200:3.0:1787866194.067032:0:68006:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.58@tcp
00000400:00000200:3.0:1787866194.067036:0:68006:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [7:7:256:4294967295] 192.168.203.58@tcp LPNI seq info [7:7:8:4294967295]
00000400:00000200:3.0:1787866194.067042:0:68006:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:3.0:1787866194.067048:0:68006:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:<?>) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : GET try# 0
00000800:00000200:3.0:1787866194.067053:0:68006:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.203.58@tcp
00000800:00000010:3.0:1787866194.067057:0:68006:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000ea7120b3.
00000800:00000200:3.0:1787866194.067077:0:68006:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000d15e6937] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:3.0:1787866194.067085:0:68006:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000200:3.0:1787866194.067089:0:68006:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ea7120b3 type 2, nob 96 niov 1 nkiov 0
00000400:00000200:3.0:1787866194.067117:0:68006:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.203.58@tcp added to recovery queue. ping count: 3 next ping: 1411 health :0
00000400:00000200:2.0:1787866194.067331:0:68016:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 3
00000400:00000200:2.0:1787866194.067337:0:68016:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.203.58@tcp(192.168.203.58@tcp)->dst 192.168.203.58@tcp: GET simulate health error: LOCAL_ERROR
00000800:02000400:2.0:1787866194.067340:0:68016:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.203.58: rc = -22
00000800:00000200:2.0:1787866194.071128:0:68016:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [00000000f783d8b3] Error -22 on write to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000010:2.0:1787866194.071138:0:68016:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000ea7120b3.
00000400:00000200:2.0:1787866194.071146:0:68016:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 5
00000400:00000200:2.0:1787866194.071151:0:68016:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: GET: LOCAL_ERROR
00000400:00000200:2.0:1787866194.071160:0:68016:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5
00000400:00000200:2.0:1787866194.071163:0:68016:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.203.58@tcp recovery message sent unsuccessfully:-5
00000400:00000200:2.0:1787866194.071166:0:68016:0:(lib-move.c:4071:lnet_handle_recovery_reply()) local NI (192.168.203.58@tcp) recovery failed with -5
00000400:00000010:0.0:1787866194.194621:0:68362:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000912c74c3.
00000400:00000010:0.0:1787866194.194796:0:68362:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000912c74c3.
00000400:00000010:2.0:1787866194.245952:0:68366:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000005b81f753.
00000400:00000010:2.0:1787866194.246055:0:68366:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000005b81f753.
00000400:00000200:3.0:1787866194.320232:0:68373:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 0
00000400:00000010:3.0:1787866194.320242:0:68373:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 0000000012f17a2c.
00000400:00000200:3.0:1787866194.320252:0:68373:0:(api-ni.c:9567:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:3.0:1787866194.320270:0:68373:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 000000002df42b09.
00000400:00000200:3.0:1787866194.320277:0:68373:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:3.0:1787866194.320282:0:68373:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 000000002df42b09.
00000400:00000010:3.0:1787866194.320290:0:68373:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 0000000012f17a2c.
00000001:02000400:3.0:1787866194.347791:0:68375:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -l
00000400:00000200:0.0:1787866194.368998:0:68376:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 0
00000400:00000010:0.0:1787866194.369007:0:68376:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000006e78d75b.
00000400:00000200:0.0:1787866194.369017:0:68376:0:(api-ni.c:9567:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:0.0:1787866194.369039:0:68376:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000305c9440.
00000400:00000200:0.0:1787866194.369046:0:68376:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:0.0:1787866194.369050:0:68376:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000305c9440.
00000400:00000010:0.0:1787866194.369059:0:68376:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000006e78d75b.
00000001:02000400:0.0:1787866194.381349:0:68377:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net_drop del -a
00000400:00000200:3.0:1787866194.401648:0:68378:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866194.401658:0:68378:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866194.401663:0:68378:0:(api-ni.c:3207:nla_strnid()) <?> -> <?>
00000400:00000200:3.0:1787866194.401665:0:68378:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866194.401666:0:68378:0:(api-ni.c:3207:nla_strnid()) <?> -> <?>
00000400:00000200:3.0:1787866194.401669:0:68378:0:(net_fault.c:254:lnet_drop_rule_del()) src <?> dst <?>
00000400:00000200:3.0:1787866194.401678:0:68378:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp->dst: 255.255.255.255@tcp (1/1, 0)
00000400:00000010:3.0:1787866194.401681:0:68378:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 0000000044c03cfa.
00000400:00000200:3.0:1787866194.401686:0:68378:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp1->dst: 255.255.255.255@tcp1 (1/1, 0)
00000400:00000010:3.0:1787866194.401688:0:68378:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 00000000b7b9bb79.
00000400:00000200:3.0:1787866194.401693:0:68378:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp->dst: 255.255.255.255@tcp (1/1, 0)
00000400:00000010:3.0:1787866194.401695:0:68378:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 0000000061201bde.
00000400:00000200:3.0:1787866194.401699:0:68378:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp1->dst: 255.255.255.255@tcp1 (1/1, 0)
00000400:00000010:3.0:1787866194.401700:0:68378:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 00000000265e7ce9.
00000001:02000400:2.0:1787866194.452360:0:68381:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
00000400:00000200:2.0:1787866194.468483:0:68382:0:(net_fault.c:254:lnet_drop_rule_del()) src <?> dst <?>
00000400:00000200:1.0:1787866194.468652:0:68007:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: -108
00000400:00000200:1.0:1787866194.468657:0:68007:0:(peer.c:4385:lnet_peer_discovery()) stopping
00000400:00000200:1.0:1787866194.468659:0:68007:0:(peer.c:4431:lnet_peer_discovery()) stopped
00000400:00000200:2.0:1787866194.468744:0:68382:0:(peer.c:4488:lnet_peer_discovery_stop()) discovery stopped
00000400:00000010:2.0:1787866194.469251:0:68382:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 00000000925783b6.
00000400:00000010:2.0:1787866194.469260:0:68382:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 000000002a00e456.
00000400:00000010:2.0:1787866194.469262:0:68382:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 000000006f559dd9.
00000400:00000010:2.0:1787866194.469264:0:68382:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 00000000d1c331d2.
00000400:00000200:2.0:1787866194.469271:0:68382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a3edd4bd
00000400:00000010:2.0:1787866194.469273:0:68382:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000a3edd4bd.
00000400:00000200:2.0:1787866194.469277:0:68382:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0
00000400:00000200:2.0:1787866194.469283:0:68382:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.203.58@tcp recovery ping unlinked
00000400:00000010:2.0:1787866194.469286:0:68382:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000c58a0ec5.
00000400:00000010:2.0:1787866194.469290:0:68382:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000099ab2b98.
00000400:00000010:2.0:1787866194.469293:0:68382:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 00000000fab4034e.
00000400:00000010:2.0:1787866194.469295:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000075b49a8d.
00000400:00000010:2.0:1787866194.469298:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000007213af63.
00000400:00000010:2.0:1787866194.469303:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000039e568ed.
00000400:00000010:2.0:1787866194.469304:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000ca6fac8d.
00000400:00000010:2.0:1787866194.469309:0:68382:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000352a51e8.
00000400:00000200:2.0:1787866194.469312:0:68382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b3cf5b6b
00000400:00000010:2.0:1787866194.469313:0:68382:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000b3cf5b6b.
00000400:00000200:2.0:1787866194.469316:0:68382:0:(api-ni.c:2393:lnet_push_target_event_handler()) type 6 status 0 unlinked 1
00000400:00000010:2.0:1787866194.469319:0:68382:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000ffa05391.
00000400:00000400:2.0:1787866194.469322:0:68382:0:(lib-ptl.c:964:lnet_clear_lazy_portal()) Active lazy portal 0 on exit
00000400:00000010:2.0:1787866194.474109:0:68382:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000073d4377f.
00000400:00000200:2.0:1787866194.474124:0:68382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000053c24a3e
00000400:00000010:2.0:1787866194.474126:0:68382:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 0000000053c24a3e.
00000400:00000010:2.0:1787866194.474134:0:68382:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 73 at 0000000005408f50.
00000400:00000200:1.0:1787866194.474273:0:68019:0:(acceptor.c:659:lnet_acceptor()) Acceptor stopping
00000400:00000200:2.0:1787866194.474394:0:68382:0:(router.c:792:lnet_del_route()) Del route: net <255:65535> : gw <?>
00000400:00000200:2.0:1787866194.474494:0:68382:0:(lo.c:49:lolnd_shutdown()) shutdown
00000400:00000010:2.0:1787866194.474497:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000ca7f81c8.
00000400:00000010:2.0:1787866194.474502:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000d74fa375.
00000400:00000010:2.0:1787866194.474504:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f6653601.
00000400:00000010:2.0:1787866194.474506:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000000b98e9f.
00000400:00000010:2.0:1787866194.474509:0:68382:0:(config.c:293:lnet_ni_free()) kfreed 'ni': 552 at 0000000092bb6b37.
00000400:00000010:2.0:1787866194.474519:0:68382:0:(config.c:329:lnet_net_free()) kfreed 'net': 176 at 00000000ea694c1b.
00000800:00000200:2.1:1787866194.474534:0:68382:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 2, ksnr_connected a ksnr_max_conns 1
00000800:00000200:2.1:1787866194.474637:0:68382:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 3, ksnr_connected 2 ksnr_max_conns 1
00000800:00000200:2.1:1787866194.474672:0:68382:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 1, ksnr_connected 2 ksnr_max_conns 1
00000800:00000200:2.1:1787866194.474705:0:68382:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 1, ksnr_connected 0 ksnr_max_conns 1
00000800:00000010:2.1:1787866194.474708:0:68382:0:(socklnd.c:126:ksocknal_destroy_conn_cb()) kfreed 'conn_cb': 192 at 0000000040eb0d5e.
00000400:00020000:2.0:1787866194.474715:0:68382:0:(acceptor.c:252:lnet_acceptor_remove_socket()) Interface ens2 not found
00000800:00000200:3.0:1787866194.480398:0:68015:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 00000000b1b59a22
00000800:00000010:3.0:1787866194.480482:0:68015:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 00000000b1b59a22.
00000800:00000200:3.0:1787866194.480487:0:68015:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 00000000f783d8b3
00000800:00000010:3.0:1787866194.480489:0:68015:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 00000000f783d8b3.
00000800:00000200:3.0:1787866194.480491:0:68015:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 00000000bba29d83
00000800:00000010:3.0:1787866194.480492:0:68015:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 00000000bba29d83.
00000800:00000200:3.0:1787866194.480494:0:68015:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 0000000087eabc9c
00000800:00000200:3.0:1787866194.480511:0:68015:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.203.58@tcp 00000000d15e6937 deleted
00000800:00000010:3.0:1787866194.480513:0:68015:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 00000000d15e6937.
00000800:00000010:3.0:1787866194.480516:0:68015:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 0000000087eabc9c.
00000800:00000200:2.0:1787866195.539396:0:68382:0:(socklnd.c:2492:ksocknal_shutdown()) waiting for 0 peers to disconnect
00000800:00000010:2.0:1787866195.539438:0:68382:0:(socklnd.c:2498:ksocknal_shutdown()) kfreed 'net': 208 at 0000000050d5d605.
00000400:02000000:2.0:1787866195.539451:0:68382:0:(api-ni.c:2563:lnet_clear_zombies_nis_locked()) Removed LNI 192.168.203.58@tcp
00000400:00000010:2.0:1787866195.546127:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000907726e0.
00000400:00000010:2.0:1787866195.546140:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000b8416bbd.
00000400:00000010:2.0:1787866195.546143:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000006efe2553.
00000400:00000010:2.0:1787866195.546145:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000a85b7985.
00000400:00000010:2.0:1787866195.546148:0:68382:0:(config.c:286:lnet_ni_free()) kfreed 'ni->ni_interface': 5 at 0000000062a90f6c.
00000400:00000010:2.0:1787866195.546151:0:68382:0:(config.c:293:lnet_ni_free()) kfreed 'ni': 552 at 0000000087d0447c.
00000400:00000200:2.0:1787866195.546299:0:68382:0:(peer.c:620:lnet_peer_ni_clr_local_net_locked()) 00000000a7dbe09a nid 192.168.203.58@tcp detached from local net
00000400:00000010:2.0:1787866195.546305:0:68382:0:(config.c:329:lnet_net_free()) kfreed 'net': 176 at 00000000e6f5303d.
00000400:00020000:2.0:1787866195.546330:0:68382:0:(acceptor.c:252:lnet_acceptor_remove_socket()) Interface ens2 not found
00000800:00000010:2.0:1787866195.559734:0:68382:0:(socklnd.c:2498:ksocknal_shutdown()) kfreed 'net': 208 at 000000000e3043f0.
00000800:00000010:2.0:1787866195.559747:0:68382:0:(socklnd.c:2235:ksocknal_base_shutdown()) before NAL cleanup: kmem 418789
00000800:00000200:2.0:1787866195.559755:0:68382:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_GOING_DOWN, iface=lo ifindex 1 state 0 ns 00000000198b8817
00000800:00000200:2.0:1787866195.559758:0:68382:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_DOWN, iface=lo ifindex 1 state 0 ns 00000000198b8817
00000800:00000200:2.0:1787866195.559760:0:68382:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UNREGISTER, iface=lo ifindex 1 state 0 ns 00000000198b8817
00000800:00000200:2.0:1787866195.559762:0:68382:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_GOING_DOWN, iface=ens2 ifindex 2 state 6 ns 00000000198b8817
00000800:00000200:2.0:1787866195.559764:0:68382:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_DOWN, iface=ens2 ifindex 2 state 6 ns 00000000198b8817
00000800:00000200:2.0:1787866195.559766:0:68382:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UNREGISTER, iface=ens2 ifindex 2 state 6 ns 00000000198b8817
00000400:00000010:3.0:1787866195.570932:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 00000000aa17dfbf.
00000400:00000010:3.0:1787866195.570942:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000008e0ed7af.
00000800:00000010:3.0:1787866195.570946:0:68382:0:(socklnd.c:2296:ksocknal_base_shutdown()) after NAL cleanup: kmem 418637
00000400:02000000:3.0:1787866195.570955:0:68382:0:(api-ni.c:2563:lnet_clear_zombies_nis_locked()) Removed LNI 192.168.203.58@tcp1
00000400:00000010:3.0:1787866195.574674:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000aafd4e56.
00000400:00000010:3.0:1787866195.574686:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000a9abb91b.
00000400:00000010:3.0:1787866195.574689:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000007620a0d3.
00000400:00000010:3.0:1787866195.574690:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000d723100b.
00000400:00000010:3.0:1787866195.574693:0:68382:0:(config.c:286:lnet_ni_free()) kfreed 'ni->ni_interface': 5 at 000000006ff262b0.
00000400:00000010:3.0:1787866195.574696:0:68382:0:(config.c:293:lnet_ni_free()) kfreed 'ni': 552 at 000000007e669fe5.
00000400:00000200:3.0:1787866195.574711:0:68382:0:(peer.c:442:lnet_peer_del_locked()) peer 192.168.203.58@tcp1
00000400:00000200:3.0:1787866195.574717:0:68382:0:(peer.c:364:lnet_peer_detach_peer_ni_locked()) peer 192.168.203.58@tcp1 NID 192.168.203.58@tcp
00000400:00000200:3.0:1787866195.574720:0:68382:0:(peer.c:2392:lnet_destroy_peer_ni_locked()) 00000000a7dbe09a nid 192.168.203.58@tcp
00000400:00000010:3.0:1787866195.574722:0:68382:0:(peer.c:2422:lnet_destroy_peer_ni_locked()) kfreed 'lpni': 480 at 00000000a7dbe09a.
00000400:00000200:3.0:1787866195.574726:0:68382:0:(peer.c:213:lnet_destroy_peer_net_locked()) 0000000060507bb8 net tcp
00000400:00000010:3.0:1787866195.574728:0:68382:0:(peer.c:220:lnet_destroy_peer_net_locked()) kfreed 'lpn': 72 at 0000000060507bb8.
00000400:00000200:3.0:1787866195.574732:0:68382:0:(peer.c:364:lnet_peer_detach_peer_ni_locked()) peer 192.168.203.58@tcp1 NID 192.168.203.58@tcp1
00000400:00000200:3.0:1787866195.574735:0:68382:0:(peer.c:2392:lnet_destroy_peer_ni_locked()) 00000000aee8cefc nid 192.168.203.58@tcp1
00000400:00000010:3.0:1787866195.574736:0:68382:0:(peer.c:2422:lnet_destroy_peer_ni_locked()) kfreed 'lpni': 480 at 00000000aee8cefc.
00000400:00000200:3.0:1787866195.574739:0:68382:0:(peer.c:213:lnet_destroy_peer_net_locked()) 000000001b28f015 net tcp1
00000400:00000010:3.0:1787866195.574740:0:68382:0:(peer.c:220:lnet_destroy_peer_net_locked()) kfreed 'lpn': 72 at 000000001b28f015.
00000400:00000200:3.0:1787866195.574743:0:68382:0:(peer.c:274:lnet_destroy_peer_locked()) 00000000665005b4 nid 192.168.203.58@tcp1
00000400:00000010:3.0:1787866195.574746:0:68382:0:(peer.c:304:lnet_destroy_peer_locked()) kfreed 'lp': 360 at 00000000665005b4.
00000400:00000010:3.0:1787866195.574751:0:68382:0:(config.c:329:lnet_net_free()) kfreed 'net': 176 at 000000000e09f060.
00000400:00000010:3.0:1787866195.574757:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000037f27f8a.
00000400:00000010:3.0:1787866195.574759:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000f23e1bfd.
00000400:00000010:3.0:1787866195.574764:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000d94bb90a.
00000400:00000010:3.0:1787866195.574768:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000004494a819.
00000400:00000010:3.0:1787866195.574770:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000d576df6d.
00000400:00000010:3.0:1787866195.574772:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000049c09a60.
00000400:00000010:3.0:1787866195.574775:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f3b8bfe7.
00000400:00000010:3.0:1787866195.574793:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000001e04f513.
00000400:00000010:3.0:1787866195.574795:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000050988b52.
00000400:00000010:3.0:1787866195.574796:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000023ff55ad.
00000400:00000010:3.0:1787866195.574799:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000002f504b20.
00000400:00000010:3.0:1787866195.574802:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000cccc9757.
00000400:00000010:3.0:1787866195.574803:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002bc5854b.
00000400:00000010:3.0:1787866195.574805:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000b4b34503.
00000400:00000010:3.0:1787866195.574808:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000dbb65273.
00000400:00000010:3.0:1787866195.574811:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000d8180f8a.
00000400:00000010:3.0:1787866195.574812:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002379337e.
00000400:00000010:3.0:1787866195.574814:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000c46746d3.
00000400:00000010:3.0:1787866195.574817:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000042dece22.
00000400:00000010:3.0:1787866195.574819:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000cea13fef.
00000400:00000010:3.0:1787866195.574820:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000078b0cb5f.
00000400:00000010:3.0:1787866195.574822:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000086becfe9.
00000400:00000010:3.0:1787866195.574824:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ede22331.
00000400:00000010:3.0:1787866195.574826:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000008be1229f.
00000400:00000010:3.0:1787866195.574828:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000a513f0c7.
00000400:00000010:3.0:1787866195.574829:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e0f4b49a.
00000400:00000010:3.0:1787866195.576426:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000874db7d2.
00000400:00000010:3.0:1787866195.576430:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000ccee0bbb.
00000400:00000010:3.0:1787866195.576432:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000022f4e0e5.
00000400:00000010:3.0:1787866195.576435:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000009f0e7ba3.
00000400:00000010:3.0:1787866195.576438:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ed353615.
00000400:00000010:3.0:1787866195.576442:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000005cf25d6b.
00000400:00000010:3.0:1787866195.576443:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000009a537483.
00000400:00000010:3.0:1787866195.576445:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000008d8322ba.
00000400:00000010:3.0:1787866195.576448:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000006e575a31.
00000400:00000010:3.0:1787866195.576451:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000b81fa609.
00000400:00000010:3.0:1787866195.576452:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000067279915.
00000400:00000010:3.0:1787866195.576453:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000a5aed5b6.
00000400:00000010:3.0:1787866195.576457:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000091bd4b17.
00000400:00000010:3.0:1787866195.576458:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000de939f20.
00000400:00000010:3.0:1787866195.576473:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000b1b9e545.
00000400:00000010:3.0:1787866195.576475:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e3cbe3fc.
00000400:00000010:3.0:1787866195.576478:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f35d978d.
00000400:00000010:3.0:1787866195.576480:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000067094ef7.
00000400:00000010:3.0:1787866195.576482:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000006f39c091.
00000400:00000010:3.0:1787866195.576483:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000007154a7e2.
00000400:00000010:3.0:1787866195.576486:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000004da4d315.
00000400:00000010:3.0:1787866195.576490:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000003d968886.
00000400:00000010:3.0:1787866195.576492:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000084ec0e37.
00000400:00000010:3.0:1787866195.576493:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000024d8becd.
00000400:00000010:3.0:1787866195.576496:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000007666a5df.
00000400:00000010:3.0:1787866195.576498:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000066d309d2.
00000400:00000010:3.0:1787866195.576500:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000436d5143.
00000400:00000010:3.0:1787866195.576501:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000003ac074f2.
00000400:00000010:3.0:1787866195.576505:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000007654107.
00000400:00000010:3.0:1787866195.576507:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000004e96b557.
00000400:00000010:3.0:1787866195.576509:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000a1903a0c.
00000400:00000010:3.0:1787866195.576510:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000001a570e11.
00000400:00000010:3.0:1787866195.576513:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000eec38694.
00000400:00000010:3.0:1787866195.576515:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000004dc61355.
00000400:00000010:3.0:1787866195.576517:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000001457018c.
00000400:00000010:3.0:1787866195.576519:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000f615b3f7.
00000400:00000010:3.0:1787866195.576523:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000af0841f6.
00000400:00000010:3.0:1787866195.576525:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000e6636fb0.
00000400:00000010:3.0:1787866195.576526:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000092c1d6d3.
00000400:00000010:3.0:1787866195.576527:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000007545239e.
00000400:00000010:3.0:1787866195.576530:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000372d9216.
00000400:00000010:3.0:1787866195.576532:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000005bad2a7b.
00000400:00000010:3.0:1787866195.576533:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000081f58040.
00000400:00000010:3.0:1787866195.576535:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000093c9fcf8.
00000400:00000010:3.0:1787866195.576538:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000066fd5643.
00000400:00000010:3.0:1787866195.576540:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000a4963b83.
00000400:00000010:3.0:1787866195.576541:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000bfd43d4d.
00000400:00000010:3.0:1787866195.576549:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000ca528c1f.
00000400:00000010:3.0:1787866195.576552:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000001fc9d250.
00000400:00000010:3.0:1787866195.576555:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000b39d4f03.
00000400:00000010:3.0:1787866195.576556:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000c408af56.
00000400:00000010:3.0:1787866195.576557:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000040d72397.
00000400:00000010:3.0:1787866195.576560:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ad1ad778.
00000400:00000010:3.0:1787866195.576562:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000009e0d68a0.
00000400:00000010:3.0:1787866195.576564:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000004d0945d9.
00000400:00000010:3.0:1787866195.576565:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000021f0813b.
00000400:00000010:3.0:1787866195.576569:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000003cb04dca.
00000400:00000010:3.0:1787866195.576571:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000004f995630.
00000400:00000010:3.0:1787866195.576573:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000cb4bd87b.
00000400:00000010:3.0:1787866195.576574:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000056efd22.
00000400:00000010:3.0:1787866195.576578:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ff0fa99f.
00000400:00000010:3.0:1787866195.576580:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000b2540f13.
00000400:00000010:3.0:1787866195.576581:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000315ac413.
00000400:00000010:3.0:1787866195.576583:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000b3612087.
00000400:00000010:3.0:1787866195.576585:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000b355e253.
00000400:00000010:3.0:1787866195.576587:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000069317189.
00000400:00000010:3.0:1787866195.576589:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000e77cd98a.
00000400:00000010:3.0:1787866195.576590:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000b0bcd302.
00000400:00000010:3.0:1787866195.576594:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000061615e9.
00000400:00000010:3.0:1787866195.576597:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000e3db74e8.
00000400:00000010:3.0:1787866195.576598:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000aeed7957.
00000400:00000010:3.0:1787866195.576600:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000a8e3b738.
00000400:00000010:3.0:1787866195.576602:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000a01446bf.
00000400:00000010:3.0:1787866195.576604:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000072564e3b.
00000400:00000010:3.0:1787866195.576606:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000b3074146.
00000400:00000010:3.0:1787866195.576607:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000007da5d9b4.
00000400:00000010:3.0:1787866195.576610:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ec41b373.
00000400:00000010:3.0:1787866195.576613:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000a710530b.
00000400:00000010:3.0:1787866195.576614:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000c10d026b.
00000400:00000010:3.0:1787866195.576616:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000ab4cbdf5.
00000400:00000010:3.0:1787866195.576757:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000002bcb5613.
00000400:00000010:3.0:1787866195.576760:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000bfac8294.
00000400:00000010:3.0:1787866195.576761:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002aa3873b.
00000400:00000010:3.0:1787866195.576763:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e9590d32.
00000400:00000010:3.0:1787866195.576767:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000b5fbf0e5.
00000400:00000010:3.0:1787866195.576769:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000009fc95cce.
00000400:00000010:3.0:1787866195.576770:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000a0fb2ddc.
00000400:00000010:3.0:1787866195.576772:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e4649c08.
00000400:00000010:3.0:1787866195.576775:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000956c410a.
00000400:00000010:3.0:1787866195.576777:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f45699e6.
00000400:00000010:3.0:1787866195.576778:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000005a3e6480.
00000400:00000010:3.0:1787866195.576780:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000b2a3b6a8.
00000400:00000010:3.0:1787866195.576783:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000003faedac8.
00000400:00000010:3.0:1787866195.576785:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000dc92e049.
00000400:00000010:3.0:1787866195.576787:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000f52ae701.
00000400:00000010:3.0:1787866195.576788:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000003ad4337c.
00000400:00000010:3.0:1787866195.576791:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000e93507ee.
00000400:00000010:3.0:1787866195.576793:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000020a8a065.
00000400:00000010:3.0:1787866195.576794:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000351f65b9.
00000400:00000010:3.0:1787866195.576796:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000012d03674.
00000400:00000010:3.0:1787866195.576799:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000005604167f.
00000400:00000010:3.0:1787866195.576801:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000090520b28.
00000400:00000010:3.0:1787866195.576803:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000046a43c45.
00000400:00000010:3.0:1787866195.576804:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000009ffd5ad9.
00000400:00000010:3.0:1787866195.576807:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000049f79d76.
00000400:00000010:3.0:1787866195.576824:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000009ec4540a.
00000400:00000010:3.0:1787866195.576826:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000077e5c6ec.
00000400:00000010:3.0:1787866195.576828:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000008787625a.
00000400:00000010:3.0:1787866195.577002:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000028b9089e.
00000400:00000010:3.0:1787866195.577007:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000303fbebc.
00000400:00000010:3.0:1787866195.577009:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002602338d.
00000400:00000010:3.0:1787866195.577012:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000016ffbbec.
00000400:00000010:3.0:1787866195.577016:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000cb480af2.
00000400:00000010:3.0:1787866195.577025:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000cfcf2778.
00000400:00000010:3.0:1787866195.577027:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000aded3c3a.
00000400:00000010:3.0:1787866195.577029:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000070635d03.
00000400:00000010:3.0:1787866195.577032:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000a27db30f.
00000400:00000010:3.0:1787866195.577034:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000003a038b54.
00000400:00000010:3.0:1787866195.577036:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000873b033d.
00000400:00000010:3.0:1787866195.577038:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000a3257d5d.
00000400:00000010:3.0:1787866195.577040:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ce65c75a.
00000400:00000010:3.0:1787866195.577043:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000bf17c8c4.
00000400:00000010:3.0:1787866195.577044:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000d6f93c62.
00000400:00000010:3.0:1787866195.577046:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000072de1828.
00000400:00000010:3.0:1787866195.577049:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000007434b255.
00000400:00000010:3.0:1787866195.577051:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000aaddbc94.
00000400:00000010:3.0:1787866195.577052:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000685aa72e.
00000400:00000010:3.0:1787866195.577053:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000072029be1.
00000400:00000010:3.0:1787866195.577056:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000076d6cd1c.
00000400:00000010:3.0:1787866195.577058:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000006a1e06d4.
00000400:00000010:3.0:1787866195.577060:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000e65ff80a.
00000400:00000010:3.0:1787866195.577062:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000a4918ca0.
00000400:00000010:3.0:1787866195.577065:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000015c7a150.
00000400:00000010:3.0:1787866195.577066:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000fc4e5ceb.
00000400:00000010:3.0:1787866195.577067:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000080b4b9f8.
00000400:00000010:3.0:1787866195.577069:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000ab15aaf2.
00000400:00000010:3.0:1787866195.577073:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000006d892fb3.
00000400:00000010:3.0:1787866195.577075:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f13b8109.
00000400:00000010:3.0:1787866195.577076:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000000d103696.
00000400:00000010:3.0:1787866195.577078:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000fd0ac38c.
00000400:00000010:3.0:1787866195.577148:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000072a8f16f.
00000400:00000010:3.0:1787866195.577152:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f1ecd397.
00000400:00000010:3.0:1787866195.577154:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000009467422c.
00000400:00000010:3.0:1787866195.577156:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000aaa9c056.
00000400:00000010:3.0:1787866195.577160:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000a3e62e9d.
00000400:00000010:3.0:1787866195.577173:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000002955af4b.
00000400:00000010:3.0:1787866195.577179:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000026f6c9c0.
00000400:00000010:3.0:1787866195.577181:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000007ae10c44.
00000400:00000010:3.0:1787866195.577184:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000799598b3.
00000400:00000010:3.0:1787866195.577186:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000ad0a5617.
00000400:00000010:3.0:1787866195.577187:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000067d2cebe.
00000400:00000010:3.0:1787866195.577189:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000be2546b6.
00000400:00000010:3.0:1787866195.577191:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000e53a5eee.
00000400:00000010:3.0:1787866195.577193:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000036360d9c.
00000400:00000010:3.0:1787866195.577195:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000001e54cbd7.
00000400:00000010:3.0:1787866195.577196:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000048d6c5d0.
00000400:00000010:3.0:1787866195.577199:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000bce067fc.
00000400:00000010:3.0:1787866195.577201:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f0975345.
00000400:00000010:3.0:1787866195.577203:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000005586a200.
00000400:00000010:3.0:1787866195.577205:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000036b3a50e.
00000400:00000010:3.0:1787866195.577208:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000067e117b8.
00000400:00000010:3.0:1787866195.577210:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000003c4758a4.
00000400:00000010:3.0:1787866195.577212:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000ef2cd2ea.
00000400:00000010:3.0:1787866195.577214:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000375896ff.
00000400:00000010:3.0:1787866195.577217:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000003dca47e3.
00000400:00000010:3.0:1787866195.577219:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000001a01547f.
00000400:00000010:3.0:1787866195.577221:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000023cd7e93.
00000400:00000010:3.0:1787866195.577222:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000012c6c579.
00000400:00000010:3.0:1787866195.577226:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000052006a4e.
00000400:00000010:3.0:1787866195.577228:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000639234ad.
00000400:00000010:3.0:1787866195.577229:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000008876f365.
00000400:00000010:3.0:1787866195.577231:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000013205a57.
00000400:00000010:3.0:1787866195.577234:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000006ce923da.
00000400:00000010:3.0:1787866195.577236:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000015c73426.
00000400:00000010:3.0:1787866195.577237:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000006af87bde.
00000400:00000010:3.0:1787866195.577239:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000007bf2fb4c.
00000400:00000010:3.0:1787866195.577242:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ff320cb3.
00000400:00000010:3.0:1787866195.577244:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000004d9a224a.
00000400:00000010:3.0:1787866195.577245:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000065590823.
00000400:00000010:3.0:1787866195.577252:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000094312786.
00000400:00000010:3.0:1787866195.577255:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000010775746.
00000400:00000010:3.0:1787866195.577257:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000001d9cf5b7.
00000400:00000010:3.0:1787866195.577258:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000001c3ad77c.
00000400:00000010:3.0:1787866195.577260:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000aa387aa5.
00000400:00000010:3.0:1787866195.577262:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000006fe7da78.
00000400:00000010:3.0:1787866195.577265:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000000a23be36.
00000400:00000010:3.0:1787866195.577266:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000000c3faf23.
00000400:00000010:3.0:1787866195.577268:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000088e7fc31.
00000400:00000010:3.0:1787866195.577270:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f68aaa16.
00000400:00000010:3.0:1787866195.577272:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000aab07357.
00000400:00000010:3.0:1787866195.577274:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000e7f6ed49.
00000400:00000010:3.0:1787866195.577275:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000c2ec0b30.
00000400:00000010:3.0:1787866195.577279:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000eff1234f.
00000400:00000010:3.0:1787866195.577293:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000003f0bdf4c.
00000400:00000010:3.0:1787866195.577295:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000ea68bc5b.
00000400:00000010:3.0:1787866195.577297:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000ae718185.
00000400:00000010:3.0:1787866195.577300:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000000b3665f6.
00000400:00000010:3.0:1787866195.577302:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000874a2026.
00000400:00000010:3.0:1787866195.577303:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000f2e30eec.
00000400:00000010:3.0:1787866195.577306:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000002d74340f.
00000400:00000010:3.0:1787866195.577309:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000003506a16b.
00000400:00000010:3.0:1787866195.577313:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000006ca9eb8.
00000400:00000010:3.0:1787866195.577315:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000fccb4c56.
00000400:00000010:3.0:1787866195.577317:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000009dea2467.
00000400:00000010:3.0:1787866195.577320:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000000ee9133.
00000400:00000010:3.0:1787866195.577323:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f9cb8dba.
00000400:00000010:3.0:1787866195.577325:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000062c72d1a.
00000400:00000010:3.0:1787866195.577326:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000019d8f1d6.
00000400:00000010:3.0:1787866195.577329:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000e4747cb8.
00000400:00000010:3.0:1787866195.577332:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000dfd99f4c.
00000400:00000010:3.0:1787866195.577333:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000d78759ab.
00000400:00000010:3.0:1787866195.577334:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000010a6e38f.
00000400:00000010:3.0:1787866195.577342:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f8e9a5a4.
00000400:00000010:3.0:1787866195.577344:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000007aed3052.
00000400:00000010:3.0:1787866195.577346:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000006ef8a308.
00000400:00000010:3.0:1787866195.577347:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000005e1ca33c.
00000400:00000010:3.0:1787866195.577349:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000079ea313f.
00000400:00000010:3.0:1787866195.577351:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000be3d3d27.
00000400:00000010:3.0:1787866195.577352:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000cd87e7a3.
00000400:00000010:3.0:1787866195.577354:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000000f2b7a98.
00000400:00000010:3.0:1787866195.577356:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000088e78596.
00000400:00000010:3.0:1787866195.577358:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000009c022a70.
00000400:00000010:3.0:1787866195.577359:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000b0e1e2cf.
00000400:00000010:3.0:1787866195.577360:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e6757c30.
00000400:00000010:3.0:1787866195.577363:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000005083cb8a.
00000400:00000010:3.0:1787866195.577366:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f9a8cb83.
00000400:00000010:3.0:1787866195.577367:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000005c3c6f51.
00000400:00000010:3.0:1787866195.577369:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000b27f351f.
00000400:00000010:3.0:1787866195.577371:0:68382:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000008198bbcd.
00000400:00000010:3.0:1787866195.577373:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000df0a1b16.
00000400:00000010:3.0:1787866195.577374:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000003e32eaa7.
00000400:00000010:3.0:1787866195.577376:0:68382:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000087818236.
00000400:00000010:3.0:1787866195.577378:0:68382:0:(lib-ptl.c:854:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals': 512 at 00000000900df17a.
00000400:00000010:3.0:1787866195.577382:0:68382:0:(api-ni.c:1205:lnet_res_container_cleanup()) kfreed 'rec->rec_lh_hash': 65536 at 00000000479850c5.
00000400:00000010:3.0:1787866195.578567:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000d61d6264.
00000400:00000010:3.0:1787866195.578575:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000eb9b3a86.
00000400:00000010:3.0:1787866195.578578:0:68382:0:(api-ni.c:1205:lnet_res_container_cleanup()) kfreed 'rec->rec_lh_hash': 65536 at 00000000dc7691bc.
00000400:00000010:3.0:1787866195.578582:0:68382:0:(lib-msg.c:1308:lnet_msg_container_cleanup()) kfreed 'container->msc_finalizers': 32 at 0000000080be1e14.
00000400:00000010:3.0:1787866195.578584:0:68382:0:(lib-msg.c:1314:lnet_msg_container_cleanup()) kfreed 'container->msc_resenders': 32 at 000000006efbdcd1.
00000400:00000010:3.0:1787866195.578587:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 00000000b23e87d0.
00000400:00000010:3.0:1787866195.578589:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000e4a9e237.
00000400:00000010:3.0:1787866195.578595:0:68382:0:(peer.c:85:lnet_peer_tables_destroy()) kfreed 'hash': 8192 at 0000000066cc6be4.
00000400:00000010:3.0:1787866195.578607:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 00000000cffd9ef1.
00000400:00000010:3.0:1787866195.578609:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000037e66e1d.
00000400:00000010:3.0:1787866195.578618:0:68382:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 00000000eb7a7e8e.
00000400:00000010:3.0:1787866195.578634:0:68382:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000009f83563a.
00000400:00000010:3.0:1787866195.578637:0:68382:0:(api-ni.c:700:lnet_destroy_remote_nets_table()) kfreed 'the_lnet.ln_remote_nets_hash': 1024 at 00000000d21aed04.
00000001:00000200:3.0:1787866195.578641:0:68382:0:(udsp.c:1126:lnet_udsp_destroy()) Destroying UDSPs in the system
00000001:02000400:0.0:1787866195.607486:0:68383:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure
00000400:00000010:3.0:1787866195.651918:0:68384:0:(api-ni.c:676:lnet_create_remote_nets_table()) alloc '(hash)': 1024 at 00000000d21aed04.
00000400:00000010:3.0:1787866195.651928:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000cd87e7a3.
00000400:00000010:3.0:1787866195.651933:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000cffd9ef1.
00000400:00000010:3.0:1787866195.651936:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000006ef8a308.
00000400:00000010:3.0:1787866195.651938:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000b23e87d0.
00000400:00000010:3.0:1787866195.651944:0:68384:0:(peer.c:109:lnet_peer_tables_create()) alloc '(hash)': 8192 at 000000008198bbcd.
00000400:00000010:3.0:1787866195.651948:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000062c72d1a.
00000400:00000010:3.0:1787866195.651951:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000aa17dfbf.
00000400:00000010:3.0:1787866195.651954:0:68384:0:(lib-msg.c:1338:lnet_msg_container_setup()) alloc '(container->msc_finalizers)': 32 at 00000000fccb4c56.
00000400:00000010:3.0:1787866195.651957:0:68384:0:(lib-msg.c:1348:lnet_msg_container_setup()) alloc '(container->msc_resenders)': 32 at 00000000f2e30eec.
00000400:00000010:3.0:1787866195.652178:0:68384:0:(api-ni.c:1227:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 0000000083486692.
00000400:00000010:3.0:1787866195.652196:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ea68bc5b.
00000400:00000010:3.0:1787866195.652200:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f9a8cb83.
00000400:00000010:3.0:1787866195.652330:0:68384:0:(api-ni.c:1227:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 00000000b0147152.
00000400:00000010:3.0:1787866195.652351:0:68384:0:(lib-ptl.c:864:lnet_portals_create()) alloc '(the_lnet.ln_portals)': 512 at 000000007c65fb77.
00000400:00000010:3.0:1787866195.652355:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000b27f351f.
00000400:00000010:3.0:1787866195.652357:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e7f6ed49.
00000400:00000010:3.0:1787866195.652359:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000be3d3d27.
00000400:00000010:3.0:1787866195.652364:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005083cb8a.
00000400:00000010:3.0:1787866195.652367:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e6757c30.
00000400:00000010:3.0:1787866195.652369:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000000c3faf23.
00000400:00000010:3.0:1787866195.652371:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007aed3052.
00000400:00000010:3.0:1787866195.652375:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f29e49df.
00000400:00000010:3.0:1787866195.652377:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000000f2b7a98.
00000400:00000010:3.0:1787866195.652378:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000586537e0.
00000400:00000010:3.0:1787866195.652380:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000dfd99f4c.
00000400:00000010:3.0:1787866195.652384:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000063414f75.
00000400:00000010:3.0:1787866195.652386:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000005e1ca33c.
00000400:00000010:3.0:1787866195.652394:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005c3c6f51.
00000400:00000010:3.0:1787866195.652407:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f9cb8dba.
00000400:00000010:3.0:1787866195.652412:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000088e78596.
00000400:00000010:3.0:1787866195.652415:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000010a6e38f.
00000400:00000010:3.0:1787866195.652417:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000003507272.
00000400:00000010:3.0:1787866195.652418:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000006ca9eb8.
00000400:00000010:3.0:1787866195.652422:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000079ea313f.
00000400:00000010:3.0:1787866195.652425:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000019d8f1d6.
00000400:00000010:3.0:1787866195.652426:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000741d98d1.
00000400:00000010:3.0:1787866195.652428:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000874a2026.
00000400:00000010:3.0:1787866195.652432:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f8e9a5a4.
00000400:00000010:3.0:1787866195.652434:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000009dea2467.
00000400:00000010:3.0:1787866195.652435:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a85b7985.
00000400:00000010:3.0:1787866195.652437:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003f0bdf4c.
00000400:00000010:3.0:1787866195.652440:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e4747cb8.
00000400:00000010:3.0:1787866195.652443:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000002d74340f.
00000400:00000010:3.0:1787866195.652444:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000b8416bbd.
00000400:00000010:3.0:1787866195.652446:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000aab07357.
00000400:00000010:3.0:1787866195.652451:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000000ee9133.
00000400:00000010:3.0:1787866195.652453:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ae718185.
00000400:00000010:3.0:1787866195.652455:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000060ac5e87.
00000400:00000010:3.0:1787866195.652457:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000a23be36.
00000400:00000010:3.0:1787866195.652460:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003506a16b.
00000400:00000010:3.0:1787866195.652463:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000c2ec0b30.
00000400:00000010:3.0:1787866195.652465:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000000b98e9f.
00000400:00000010:3.0:1787866195.652466:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001d9cf5b7.
00000400:00000010:3.0:1787866195.652470:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000000b3665f6.
00000400:00000010:3.0:1787866195.652473:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000088e7fc31.
00000400:00000010:3.0:1787866195.652475:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d74fa375.
00000400:00000010:3.0:1787866195.652477:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004d9a224a.
00000400:00000010:3.0:1787866195.652480:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000eff1234f.
00000400:00000010:3.0:1787866195.652483:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000aa387aa5.
00000400:00000010:3.0:1787866195.652491:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000007213af63.
00000400:00000010:3.0:1787866195.652493:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000015c73426.
00000400:00000010:3.0:1787866195.652499:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f35d978d.
00000400:00000010:3.0:1787866195.652501:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000094312786.
00000400:00000010:3.0:1787866195.652503:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000c58a0ec5.
00000400:00000010:3.0:1787866195.652505:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000639234ad.
00000400:00000010:3.0:1787866195.652509:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000091bd4b17.
00000400:00000010:3.0:1787866195.652512:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000007bf2fb4c.
00000400:00000010:3.0:1787866195.652513:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005d1d0d9f.
00000400:00000010:3.0:1787866195.652514:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001a01547f.
00000400:00000010:3.0:1787866195.652518:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006e575a31.
00000400:00000010:3.0:1787866195.652521:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000013205a57.
00000400:00000010:3.0:1787866195.652523:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000083988c41.
00000400:00000010:3.0:1787866195.652524:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003c4758a4.
00000400:00000010:3.0:1787866195.652528:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ed353615.
00000400:00000010:3.0:1787866195.652531:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000012c6c579.
00000400:00000010:3.0:1787866195.652533:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000097374f24.
00000400:00000010:3.0:1787866195.652534:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000037f27f8a.
00000400:00000010:3.0:1787866195.652539:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000874db7d2.
00000400:00000010:3.0:1787866195.652542:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000375896ff.
00000400:00000010:3.0:1787866195.652544:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000175db05b.
00000400:00000010:3.0:1787866195.652546:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000012f17a2c.
00000400:00000010:3.0:1787866195.652550:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ede22331.
00000400:00000010:3.0:1787866195.652553:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000036b3a50e.
00000400:00000010:3.0:1787866195.652555:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000fcd82b23.
00000400:00000010:3.0:1787866195.652557:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002df42b09.
00000400:00000010:3.0:1787866195.652561:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000042dece22.
00000400:00000010:3.0:1787866195.652563:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000048d6c5d0.
00000400:00000010:3.0:1787866195.652566:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000f439e6df.
00000400:00000010:3.0:1787866195.652568:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008647e19c.
00000400:00000010:3.0:1787866195.652572:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000dbb65273.
00000400:00000010:3.0:1787866195.652574:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000be2546b6.
00000400:00000010:3.0:1787866195.652582:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000442aa5e4.
00000400:00000010:3.0:1787866195.652584:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d42ef1be.
00000400:00000010:3.0:1787866195.652588:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b355e253.
00000400:00000010:3.0:1787866195.652592:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000007ae10c44.
00000400:00000010:3.0:1787866195.652593:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001f9bae5c.
00000400:00000010:3.0:1787866195.652595:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000667cff50.
00000400:00000010:3.0:1787866195.652598:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ff0fa99f.
00000400:00000010:3.0:1787866195.652601:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000aaa9c056.
00000400:00000010:3.0:1787866195.652603:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000000874e15d.
00000400:00000010:3.0:1787866195.652604:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e4bb642e.
00000400:00000010:3.0:1787866195.652608:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003cb04dca.
00000400:00000010:3.0:1787866195.652610:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000fd0ac38c.
00000400:00000010:3.0:1787866195.652612:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000004d802733.
00000400:00000010:3.0:1787866195.652614:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006c810cfd.
00000400:00000010:3.0:1787866195.652618:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ad1ad778.
00000400:00000010:3.0:1787866195.652621:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ab15aaf2.
00000400:00000010:3.0:1787866195.652623:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000170eda37.
00000400:00000010:3.0:1787866195.652625:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005978b3b9.
00000400:00000010:3.0:1787866195.652629:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001fc9d250.
00000400:00000010:3.0:1787866195.652632:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a4918ca0.
00000400:00000010:3.0:1787866195.652633:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000007e493f7e.
00000400:00000010:3.0:1787866195.652635:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000095ccba5d.
00000400:00000010:3.0:1787866195.652639:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000066fd5643.
00000400:00000010:3.0:1787866195.652641:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000072029be1.
00000400:00000010:3.0:1787866195.652643:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e1fe24dc.
00000400:00000010:3.0:1787866195.652645:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f579002a.
00000400:00000010:3.0:1787866195.652648:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000372d9216.
00000400:00000010:3.0:1787866195.652651:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000072de1828.
00000400:00000010:3.0:1787866195.652653:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000007d9f1444.
00000400:00000010:3.0:1787866195.652654:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002a40b4c5.
00000400:00000010:3.0:1787866195.652658:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000af0841f6.
00000400:00000010:3.0:1787866195.652660:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a3257d5d.
00000400:00000010:3.0:1787866195.652667:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000040339714.
00000400:00000010:3.0:1787866195.652669:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e399c7d7.
00000400:00000010:3.0:1787866195.652674:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000eec38694.
00000400:00000010:3.0:1787866195.652676:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000070635d03.
00000400:00000010:3.0:1787866195.652679:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000006c8b3f32.
00000400:00000010:3.0:1787866195.652680:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000060bf75e9.
00000400:00000010:3.0:1787866195.652684:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000007654107.
00000400:00000010:3.0:1787866195.652686:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000016ffbbec.
00000400:00000010:3.0:1787866195.652687:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e143976c.
00000400:00000010:3.0:1787866195.652689:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005ee7d883.
00000400:00000010:3.0:1787866195.652692:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007666a5df.
00000400:00000010:3.0:1787866195.652695:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000008787625a.
00000400:00000010:3.0:1787866195.652696:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000006bf21ed9.
00000400:00000010:3.0:1787866195.652698:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006f7734d5.
00000400:00000010:3.0:1787866195.652702:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004da4d315.
00000400:00000010:3.0:1787866195.652705:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000009ffd5ad9.
00000400:00000010:3.0:1787866195.652706:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002d0d725f.
00000400:00000010:3.0:1787866195.652708:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002d1b0c79.
00000400:00000010:3.0:1787866195.652712:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005604167f.
00000400:00000010:3.0:1787866195.652714:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000012d03674.
00000400:00000010:3.0:1787866195.652716:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e20bfbc2.
00000400:00000010:3.0:1787866195.652717:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000000c2cb86.
00000400:00000010:3.0:1787866195.652722:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000028b9089e.
00000400:00000010:3.0:1787866195.652724:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000003ad4337c.
00000400:00000010:3.0:1787866195.652726:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005eb9e1b7.
00000400:00000010:3.0:1787866195.652728:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000066728e94.
00000400:00000010:3.0:1787866195.652731:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000049f79d76.
00000400:00000010:3.0:1787866195.652734:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000b2a3b6a8.
00000400:00000010:3.0:1787866195.652736:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000cf73c735.
00000400:00000010:3.0:1787866195.652737:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000dfde04a9.
00000400:00000010:3.0:1787866195.652750:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002f504b20.
00000400:00000010:3.0:1787866195.652755:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e4649c08.
00000400:00000010:3.0:1787866195.652761:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000086cf63a3.
00000400:00000010:3.0:1787866195.652763:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000087d0ea8e.
00000400:00000010:3.0:1787866195.652768:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f3b8bfe7.
00000400:00000010:3.0:1787866195.652770:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e9590d32.
00000400:00000010:3.0:1787866195.652772:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e0038e5b.
00000400:00000010:3.0:1787866195.652774:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006efe2553.
00000400:00000010:3.0:1787866195.652778:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000066cc6be4.
00000400:00000010:3.0:1787866195.652781:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000938cb033.
00000400:00000010:3.0:1787866195.652783:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d4a008b6.
00000400:00000010:3.0:1787866195.652784:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000907726e0.
00000400:00000010:3.0:1787866195.652788:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d94bb90a.
00000400:00000010:3.0:1787866195.652792:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000cc56ef7a.
00000400:00000010:3.0:1787866195.652793:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000000861544c.
00000400:00000010:3.0:1787866195.652795:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f6653601.
00000400:00000010:3.0:1787866195.652805:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ff320cb3.
00000400:00000010:3.0:1787866195.652810:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ab4cbdf5.
00000400:00000010:3.0:1787866195.652811:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000053233d68.
00000400:00000010:3.0:1787866195.652813:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ca7f81c8.
00000400:00000010:3.0:1787866195.652817:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000010775746.
00000400:00000010:3.0:1787866195.652820:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000007da5d9b4.
00000400:00000010:3.0:1787866195.652821:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000971cb4da.
00000400:00000010:3.0:1787866195.652823:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000039e568ed.
00000400:00000010:3.0:1787866195.652826:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006fe7da78.
00000400:00000010:3.0:1787866195.652829:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a8e3b738.
00000400:00000010:3.0:1787866195.652945:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000c4ab0945.
00000400:00000010:3.0:1787866195.652948:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000075b49a8d.
00000400:00000010:3.0:1787866195.652952:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f68aaa16.
00000400:00000010:3.0:1787866195.652955:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000b0bcd302.
00000400:00000010:3.0:1787866195.652957:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005736ae64.
00000400:00000010:3.0:1787866195.652959:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b90a4287.
00000400:00000010:3.0:1787866195.652970:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006ce923da.
00000400:00000010:3.0:1787866195.652974:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000b3612087.
00000400:00000010:3.0:1787866195.652980:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000003dc95ae.
00000400:00000010:3.0:1787866195.652982:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002d0f2d30.
00000400:00000010:3.0:1787866195.652986:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000067e117b8.
00000400:00000010:3.0:1787866195.658027:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000056efd22.
00000400:00000010:3.0:1787866195.658184:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000004b6bbae8.
00000400:00000010:3.0:1787866195.658190:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d9b6c465.
00000400:00000010:3.0:1787866195.658196:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003dca47e3.
00000400:00000010:3.0:1787866195.658199:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000021f0813b.
00000400:00000010:3.0:1787866195.658210:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000069363a92.
00000400:00000010:3.0:1787866195.658212:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f95cda80.
00000400:00000010:3.0:1787866195.658217:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000052006a4e.
00000400:00000010:3.0:1787866195.658219:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000040d72397.
00000400:00000010:3.0:1787866195.658221:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d8a71836.
00000400:00000010:3.0:1787866195.658223:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000095220e4a.
00000400:00000010:3.0:1787866195.658244:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a3e62e9d.
00000400:00000010:3.0:1787866195.658248:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ca528c1f.
00000400:00000010:3.0:1787866195.658250:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000345411b6.
00000400:00000010:3.0:1787866195.658252:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000054555ad8.
00000400:00000010:3.0:1787866195.658255:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000799598b3.
00000400:00000010:3.0:1787866195.658258:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000093c9fcf8.
00000400:00000010:3.0:1787866195.658259:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ce64a59b.
00000400:00000010:3.0:1787866195.658261:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000027b2bdf8.
00000400:00000010:3.0:1787866195.658265:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e53a5eee.
00000400:00000010:3.0:1787866195.658267:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000007545239e.
00000400:00000010:3.0:1787866195.658269:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000c9a4fd72.
00000400:00000010:3.0:1787866195.658271:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005b139ed2.
00000400:00000010:3.0:1787866195.658275:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000bce067fc.
00000400:00000010:3.0:1787866195.658277:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000f615b3f7.
00000400:00000010:3.0:1787866195.658279:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000943f16cf.
00000400:00000010:3.0:1787866195.658281:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000021031acc.
00000400:00000010:3.0:1787866195.658290:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006d892fb3.
00000400:00000010:3.0:1787866195.658294:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000001a570e11.
00000400:00000010:3.0:1787866195.658306:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001c3ad77c.
00000400:00000010:3.0:1787866195.658307:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e80ae833.
00000400:00000010:3.0:1787866195.658312:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000072a8f16f.
00000400:00000010:3.0:1787866195.658314:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000003ac074f2.
00000400:00000010:3.0:1787866195.658316:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000065590823.
00000400:00000010:3.0:1787866195.658318:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d28068b0.
00000400:00000010:3.0:1787866195.658322:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000076d6cd1c.
00000400:00000010:3.0:1787866195.658324:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000024d8becd.
00000400:00000010:3.0:1787866195.658325:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000006af87bde.
00000400:00000010:3.0:1787866195.658327:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007c81eb09.
00000400:00000010:3.0:1787866195.658332:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000015c7a150.
00000400:00000010:3.0:1787866195.658334:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000007154a7e2.
00000400:00000010:3.0:1787866195.658336:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000008876f365.
00000400:00000010:3.0:1787866195.658338:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d7a60416.
00000400:00000010:3.0:1787866195.658348:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007434b255.
00000400:00000010:3.0:1787866195.658351:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e3cbe3fc.
00000400:00000010:3.0:1787866195.658353:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000023cd7e93.
00000400:00000010:3.0:1787866195.658355:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000055425607.
00000400:00000010:3.0:1787866195.658359:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000cb480af2.
00000400:00000010:3.0:1787866195.658361:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a5aed5b6.
00000400:00000010:3.0:1787866195.658363:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ef2cd2ea.
00000400:00000010:3.0:1787866195.658365:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000aa4f13b0.
00000400:00000010:3.0:1787866195.658369:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a27db30f.
00000400:00000010:3.0:1787866195.658372:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000008d8322ba.
00000400:00000010:3.0:1787866195.658373:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005586a200.
00000400:00000010:3.0:1787866195.658375:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a17cc9d9.
00000400:00000010:3.0:1787866195.658378:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ce65c75a.
00000400:00000010:3.0:1787866195.658380:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000009f0e7ba3.
00000400:00000010:3.0:1787866195.658382:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001e54cbd7.
00000400:00000010:3.0:1787866195.658384:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000083953ed1.
00000400:00000010:3.0:1787866195.658393:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a01446bf.
00000400:00000010:3.0:1787866195.658409:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e0f4b49a.
00000400:00000010:3.0:1787866195.658416:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000026f6c9c0.
00000400:00000010:3.0:1787866195.658418:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e3093529.
00000400:00000010:3.0:1787866195.658422:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000061615e9.
00000400:00000010:3.0:1787866195.658425:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000086becfe9.
00000400:00000010:3.0:1787866195.658426:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000009467422c.
00000400:00000010:3.0:1787866195.658428:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000030c6bb9b.
00000400:00000010:3.0:1787866195.658431:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002bcb5613.
00000400:00000010:3.0:1787866195.658434:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000c46746d3.
00000400:00000010:3.0:1787866195.658438:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000cab59421.
00000400:00000010:3.0:1787866195.658439:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000005608000.
00000400:00000010:3.0:1787866195.658443:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ec41b373.
00000400:00000010:3.0:1787866195.658445:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000b4b34503.
00000400:00000010:3.0:1787866195.658447:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000004dc43276.
00000400:00000010:3.0:1787866195.658449:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c4efb49e.
00000400:00000010:3.0:1787866195.658459:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005d29569c.
00000400:00000010:3.0:1787866195.658462:0:68384:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000023ff55ad.
00000400:00000010:3.0:1787866195.658464:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000de4d9430.
00000400:00000010:3.0:1787866195.658466:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001af34ebb.
00000400:00000010:3.0:1787866195.658470:0:68384:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000085e71ae7.
00000400:00000010:3.0:1787866195.658474:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e3073140.
00000400:00000010:3.0:1787866195.658475:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000738b1d36.
00000400:00000010:3.0:1787866195.658481:0:68384:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 00000000a1110acd.
00000400:00000010:3.0:1787866195.658485:0:68384:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 552 at 00000000c3a2a8bc.
00000400:00000010:3.0:1787866195.658488:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000009ac881c6.
00000400:00000010:3.0:1787866195.658490:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c06af4da.
00000400:00000010:3.0:1787866195.658492:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000006e65badb.
00000400:00000010:3.0:1787866195.658494:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000bd77456a.
00000400:00000010:3.0:1787866195.658509:0:68384:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 41 at 00000000e294626b.
00000400:00000010:3.0:1787866195.658515:0:68384:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b3af47a7.
00000400:00000010:3.0:1787866195.658529:0:68384:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000df38b1fd.
00000400:00000200:3.0:1787866195.658538:0:68384:0:(lib-ptl.c:918:LNetSetLazyPortal()) Setting portal 0 lazy
00000400:00000010:3.0:1787866195.658541:0:68384:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000044c03cfa.
00000400:00000010:3.0:1787866195.658544:0:68384:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000fac06bd7.
00000400:00000010:3.0:1787866195.658551:0:68384:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000635f5135.
00000400:00000200:3.0:1787866195.658553:0:68384:0:(api-ni.c:2383:lnet_push_target_post()) posted push target 0000000044c03cfa
00000400:00000200:3.0:1787866195.658555:0:68384:0:(api-ni.c:2340:lnet_push_target_resize()) nbytes 272 success
00000400:00000010:3.0:1787866195.658558:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001cdd4679.
00000400:00000010:3.0:1787866195.658560:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000055791137.
00000400:00000010:3.0:1787866195.658562:0:68384:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005c5b9a21.
00000400:00000010:3.0:1787866195.658564:0:68384:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b840e06f.
00000400:00000200:3.0:1787866195.662177:0:68384:0:(peer.c:4457:lnet_peer_discovery_start()) discovery start: 0
00000400:00000200:0.0:1787866195.663207:0:68386:0:(peer.c:4283:lnet_peer_discovery()) started
00000400:00000010:1.0:1787866195.679749:0:68387:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000001f6718ce.
00000400:00000010:1.0:1787866195.679780:0:68387:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000001f6718ce.
00000400:00000010:1.0:1787866195.683393:0:68387:0:(api-ni.c:7409:lnet_route_show_start()) alloc '(rlist)': 16 at 000000008e7852ff.
00000400:00000010:1.0:1787866195.683426:0:68387:0:(api-ni.c:7309:lnet_route_show_done()) kfreed 'rlist': 16 at 000000008e7852ff.
00000400:00000010:1.0:1787866195.683552:0:68387:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 236 at 000000009dab4a5c.
00000400:00000010:1.0:1787866195.683560:0:68387:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 236 at 000000009dab4a5c.
00000400:00000010:1.0:1787866195.683713:0:68387:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 000000008e7852ff.
00000400:00000010:1.0:1787866195.683725:0:68387:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 000000008e7852ff.
00000400:00000010:1.0:1787866195.683796:0:68387:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 12 at 000000008e7852ff.
00000400:00000010:1.0:1787866195.683800:0:68387:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 12 at 000000008e7852ff.
00000400:00000010:1.0:1787866195.689361:0:68387:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 32 at 0000000046ab39a1.
00000001:00000200:1.0:1787866195.689371:0:68387:0:(udsp.c:863:lnet_udsp_get_policy()) Get UDSP at idx = 0
00000001:00000200:1.0:1787866195.689374:0:68387:0:(udsp.c:878:lnet_udsp_get_policy()) Found UDSP (00000000fad3d7f9)
00000400:00000010:1.0:1787866195.689377:0:68387:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 32 at 0000000046ab39a1.
00000001:02000400:2.0:1787866195.713546:0:68388:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp --if ens2
00000400:00000010:3.0:1787866195.745277:0:68389:0:(api-ni.c:6877:lnet_net_cmd()) alloc '(conf)': 808 at 00000000c4ff8df5.
00000400:00000010:3.0:1787866195.745295:0:68389:0:(api-ni.c:6555:lnet_genl_parse_local_ni()) alloc '(tun)': 172 at 00000000cb837365.
00000400:00000010:3.0:1787866195.745305:0:68389:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 00000000d8ae2599.
00000400:00000010:3.0:1787866195.745309:0:68389:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 552 at 0000000087d0447c.
00000400:00000010:3.0:1787866195.745313:0:68389:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000006110df10.
00000400:00000010:3.0:1787866195.745318:0:68389:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001d65a08f.
00000400:00000010:3.0:1787866195.745319:0:68389:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000f84b69db.
00000400:00000010:3.0:1787866195.745320:0:68389:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000bd940ac1.
00000400:00000010:3.0:1787866195.745323:0:68389:0:(config.c:397:lnet_ni_add_interface()) alloc '(ni->ni_interface)': 5 at 0000000066b316e0.
00000400:00000010:3.0:1787866195.745331:0:68389:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000062c8fb52.
00000400:00000010:3.0:1787866195.745333:0:68389:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 000000004eef61b3.
00000800:00000200:0.0:1787866195.749280:0:68389:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=lo ifindex 1 state 0 ns 00000000198b8817
00000800:00000200:0.0:1787866195.749290:0:68389:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=lo ifindex 1 state 0 ns 00000000198b8817
00000800:00000200:0.0:1787866195.749292:0:68389:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=ens2 ifindex 2 state 6 ns 00000000198b8817
00000800:00000200:0.0:1787866195.749294:0:68389:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=ens2 ifindex 2 state 6 ns 00000000198b8817
00000800:00000010:0.0:1787866195.749303:0:68389:0:(socklnd.c:2621:ksocknal_startup()) alloc '(net)': 208 at 00000000ada1255e.
00000800:00000200:0.0:1787866195.749316:0:68389:0:(socklnd_modparams.c:295:ksocklnd_lookup_conns_per_peer()) intf ens2 speed -1
00000400:00000200:3.0:1787866195.750110:0:68389:0:(config.c:1657:lnet_get_link_status_locked()) device idx 1 get_link 2
00000400:00000200:3.0:1787866195.750175:0:68389:0:(acceptor.c:209:lnet_acceptor_add_socket()) Added socket on ens2:988 (IP: 192.168.203.58)
00000400:02000000:3.0:1787866195.750188:0:68389:0:(api-ni.c:2823:lnet_startup_lndni()) Added LNI 192.168.203.58@tcp [8/256/0/180]
00000400:00000010:3.0:1787866195.758417:0:68389:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 57 at 0000000058ad283f.
00000400:00000010:3.0:1787866195.758432:0:68389:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000074a5f7bf.
00000400:00000010:3.0:1787866195.758438:0:68389:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000005a448e26.
00000400:00000010:3.0:1787866195.760010:0:68389:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b3af47a7.
00000400:00000200:3.0:1787866195.760018:0:68389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000df38b1fd
00000400:00000010:3.0:1787866195.760020:0:68389:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000df38b1fd.
00000400:00000010:3.0:1787866195.760025:0:68389:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 41 at 00000000e294626b.
00000400:00000010:3.0:1787866195.760035:0:68389:0:(api-ni.c:6852:lnet_genl_parse_local_ni()) kfreed 'tun': 172 at 00000000cb837365.
00000400:00000010:3.0:1787866195.760040:0:68389:0:(api-ni.c:6994:lnet_net_cmd()) kfreed 'conf': 808 at 00000000c4ff8df5.
00000400:00000010:2.0:1787866195.840117:0:68403:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000402763fd.
00000400:00000010:2.0:1787866195.840181:0:68403:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000402763fd.
00000001:02000400:0.0:1787866196.541575:0:68509:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp1 --if ens2
00000400:00000010:0.0:1787866196.563214:0:68510:0:(api-ni.c:6877:lnet_net_cmd()) alloc '(conf)': 808 at 00000000ca2404d1.
00000400:00000010:0.0:1787866196.563232:0:68510:0:(api-ni.c:6555:lnet_genl_parse_local_ni()) alloc '(tun)': 172 at 0000000065399e96.
00000400:00000010:0.0:1787866196.563241:0:68510:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 00000000942431d4.
00000400:00000010:0.0:1787866196.563245:0:68510:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 552 at 00000000ddcffa3a.
00000400:00000010:0.0:1787866196.563248:0:68510:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000b27d37fd.
00000400:00000010:0.0:1787866196.563252:0:68510:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009c0bc3d7.
00000400:00000010:0.0:1787866196.563254:0:68510:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000004742348e.
00000400:00000010:0.0:1787866196.563256:0:68510:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002b9ed566.
00000400:00000010:0.0:1787866196.563260:0:68510:0:(config.c:397:lnet_ni_add_interface()) alloc '(ni->ni_interface)': 5 at 000000002ea4affc.
00000800:00000010:0.0:1787866196.563267:0:68510:0:(socklnd.c:2621:ksocknal_startup()) alloc '(net)': 208 at 000000005d82d75f.
00000800:00000200:0.0:1787866196.563277:0:68510:0:(socklnd_modparams.c:295:ksocklnd_lookup_conns_per_peer()) intf ens2 speed -1
00000400:00000200:0.0:1787866196.563283:0:68510:0:(config.c:1657:lnet_get_link_status_locked()) device idx 1 get_link 2
00000400:00000200:0.0:1787866196.563323:0:68510:0:(acceptor.c:209:lnet_acceptor_add_socket()) Added socket on ens2:988 (IP: 192.168.203.58)
00000400:02000000:0.0:1787866196.563553:0:68510:0:(api-ni.c:2823:lnet_startup_lndni()) Added LNI 192.168.203.58@tcp1 [8/256/0/180]
00000400:00000010:1.0:1787866196.575704:0:68510:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 73 at 000000007148613d.
00000400:00000010:1.0:1787866196.575718:0:68510:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000001cbfee5.
00000400:00000010:1.0:1787866196.575740:0:68510:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ece06eaf.
00000400:00000010:1.0:1787866196.575757:0:68510:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000074a5f7bf.
00000400:00000200:1.0:1787866196.575761:0:68510:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000005a448e26
00000400:00000010:1.0:1787866196.575769:0:68510:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000005a448e26.
00000400:00000010:1.0:1787866196.575773:0:68510:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 57 at 0000000058ad283f.
00000400:00000010:1.0:1787866196.575851:0:68510:0:(api-ni.c:6852:lnet_genl_parse_local_ni()) kfreed 'tun': 172 at 0000000065399e96.
00000400:00000010:1.0:1787866196.575857:0:68510:0:(api-ni.c:6994:lnet_net_cmd()) kfreed 'conf': 808 at 00000000ca2404d1.
00000400:00000010:1.0:1787866196.658271:0:68515:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000001f6718ce.
00000400:00000010:1.0:1787866196.658340:0:68515:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000001f6718ce.
00000400:00000010:0.0:1787866197.510748:0:68633:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000f2fdfffa.
00000400:00000010:0.0:1787866197.511175:0:68633:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000f2fdfffa.
00000001:02000400:1.0:1787866197.538379:0:68635:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl discover 192.168.203.58@tcp
00000400:00000010:3.0:1787866197.560861:0:68636:0:(peer.c:230:lnet_peer_alloc()) alloc '(lp)': 360 at 0000000099ab2b98.
00000400:00000200:3.0:1787866197.560877:0:68636:0:(peer.c:266:lnet_peer_alloc()) 0000000099ab2b98 nid 192.168.203.58@tcp
00000400:00000010:3.0:1787866197.560883:0:68636:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 0000000018fba111.
00000400:00000200:3.0:1787866197.560891:0:68636:0:(peer.c:203:lnet_peer_net_alloc()) 0000000018fba111 net tcp
00000400:00000010:3.0:1787866197.560894:0:68636:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 00000000cb8e99f4.
00000400:00000200:3.0:1787866197.560897:0:68636:0:(peer.c:184:lnet_peer_ni_alloc()) 00000000cb8e99f4 nid 192.168.203.58@tcp
00000400:00000200:3.0:1787866197.560904:0:68636:0:(peer.c:1804:lnet_peer_attach_peer_ni()) peer 192.168.203.58@tcp NID 192.168.203.58@tcp flags 0x0
00000400:00000200:3.0:1787866197.560908:0:68636:0:(peer.c:2183:lnet_peer_ni_traffic_add()) peer 192.168.203.58@tcp: 0
00000400:00000200:3.0:1787866197.560923:0:68636:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.203.58@tcp: 0
00000400:00000200:3.0:1787866197.560925:0:68636:0:(peer.c:2935:lnet_discover_peer_locked()) Discovery attempt # 1
00000400:00000200:0.0:1787866197.561315:0:68386:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0
00000400:00000200:0.0:1787866197.561328:0:68386:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.203.58@tcp(0000000099ab2b98) state 0x40
00000400:00000010:0.0:1787866197.561334:0:68386:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000129c96e9.
00000400:00000010:0.0:1787866197.561351:0:68386:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000094dcefe0.
00000400:00000010:0.0:1787866197.561371:0:68386:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000833e52df.
00000400:00000200:0.0:1787866197.561376:0:68386:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.203.58@tcp
00000400:00000200:0.0:1787866197.561397:0:68386:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR:  192.168.203.58@tcp local destination
00000400:00000200:0.0:1787866197.561404:0:68386:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.58@tcp
00000400:00000200:0.0:1787866197.561406:0:68386:0:(lib-move.c:1874:lnet_set_non_mr_pref_nid()) Setting preferred local NID 192.168.203.58@tcp on NMR peer 192.168.203.58@tcp
00000400:00000200:0.0:1787866197.561410:0:68386:0:(peer.c:1171:lnet_peer_ni_set_non_mr_pref_nid()) peer 192.168.203.58@tcp nid 192.168.203.58@tcp: 0
00000400:00000200:0.0:1787866197.561414:0:68386:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [8:8:256:4294967295] 192.168.203.58@tcp LPNI seq info [1:1:8:4294967295]
00000400:00000200:0.0:1787866197.561419:0:68386:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:0.0:1787866197.561425:0:68386:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:<?>) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : GET try# 0
00000800:00000200:0.0:1787866197.561430:0:68386:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.203.58@tcp
00000800:00000010:0.0:1787866197.561435:0:68386:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000123fca19.
00000800:00000010:0.0:1787866197.561443:0:68386:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 0000000042229199.
00000800:00000010:0.0:1787866197.561447:0:68386:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 000000000be8e223.
00000800:00000200:0.0:1787866197.561451:0:68386:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000042229199] -> 12345-192.168.203.58@tcp (2)
00000800:00000200:0.1:1787866197.561456:0:68386:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000042229199] -> 12345-192.168.203.58@tcp (2)
00000400:00000200:0.0:1787866197.561475:0:68386:0:(peer.c:4022:lnet_peer_send_ping()) peer 192.168.203.58@tcp
00000400:00000200:0.0:1787866197.561477:0:68386:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.203.58@tcp(0000000099ab2b98) state 0x240 rc 0
00000800:00000010:1.0:1787866197.563731:0:68390:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 00000000751322ba.
00000800:00000010:1.0:1787866197.563741:0:68390:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 0000000062b5bb53.
00000800:00000010:1.0:1787866197.563747:0:68390:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000970d892d.
00000800:00000010:1.0:1787866197.563774:0:68390:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000970d892d.
00000400:00000200:2.0:1787866197.563864:0:68398:0:(acceptor.c:578:lnet_acceptor()) Accepted connection on ens2:988
00000400:00000200:2.0:1787866197.563882:0:68398:0:(acceptor.c:502:lnet_accept()) Accept 192.168.203.58@tcp from 255.3.0.2
00000800:00000010:2.0:1787866197.563886:0:68398:0:(socklnd.c:737:ksocknal_accept()) alloc '(cr)': 32 at 00000000f123ceb5.
00000800:00000010:0.0:1787866197.563924:0:68391:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 000000008d8cdc37.
00000800:00000010:0.0:1787866197.563930:0:68391:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 0000000043e38d15.
00000800:00000010:0.0:1787866197.563947:0:68391:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 00000000e6f5303d.
00000800:00000200:0.1:1787866197.563952:0:68391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000042229199] -> 12345-192.168.203.58@tcp (2)
00000800:00000200:0.1:1787866197.563957:0:68391:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.203.58@tcp 00000000e6f5303d deleted
00000800:00000010:0.1:1787866197.563959:0:68391:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 00000000e6f5303d.
00000800:00000200:0.1:1787866197.563971:0:68391:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1
00000800:00000200:0.1:1787866197.563978:0:68391:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000200:0.1:1787866197.563983:0:68391:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000123fca19 type 2, nob 96 niov 1 nkiov 0
00000800:00000200:0.0:1787866197.563988:0:68391:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.203.58@tcp p 3.x 192.168.203.58 -> 192.168.203.58:1023 incarnation:1787866195749305484 sched[0]
00000800:00000010:0.0:1787866197.563993:0:68391:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 0000000036d26412.
00000800:00000010:0.0:1787866197.564041:0:68391:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 0000000036d26412.
00000800:00000010:0.0:1787866197.564045:0:68391:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 0000000043e38d15.
00000800:00000010:0.0:1787866197.564075:0:68391:0:(socklnd_cb.c:2179:ksocknal_connd()) kfreed 'cr': 32 at 00000000f123ceb5.
00000800:00000200:2.0:1787866197.564163:0:68396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1787866197.564167:0:68396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000123fca19.
00000400:00000200:2.0:1787866197.564171:0:68396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1787866197.564177:0:68396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: GET: OK
00000400:00000200:2.0:1787866197.564182:0:68396:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 5
00000400:00000200:2.0:1787866197.564185:0:68396:0:(peer.c:3241:lnet_discovery_event_send()) Ping Send to 192.168.203.58@tcp: 0
00000800:00000200:1.1:1787866197.564688:0:68390:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1
00000800:00000200:1.0:1787866197.564697:0:68390:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.203.58@tcp p 3.x 192.168.203.58 -> 192.168.203.58:988 incarnation:1787866195749305484 sched[0]
00000800:00000010:1.0:1787866197.564701:0:68390:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 0000000062b5bb53.
00000400:00000200:1.0:1787866197.564775:0:68390:0:(lib-socket.c:438:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.203.58:988
00000400:00000200:3.0:1787866197.565127:0:68395:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : GET - for me
00000400:00000200:3.0:1787866197.565155:0:68395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.58@tcp of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:3.0:1787866197.565163:0:68395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.203.58@tcp of length 64/272 into md 0xd [1] + 0
00000400:00000200:3.0:1787866197.565176:0:68395:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.203.58@tcp to NMR:  192.168.203.58@tcp local destination
00000400:00000200:3.0:1787866197.565181:0:68395:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [9:9:256:4294967295] 192.168.203.58@tcp LPNI seq info [2:2:8:4294967295]
00000400:00000200:3.0:1787866197.565190:0:68395:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) 192.168.203.58@tcp : REPLY try# 0
00000800:00000200:3.0:1787866197.565194:0:68395:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.203.58@tcp
00000800:00000010:3.0:1787866197.565200:0:68395:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000ea7120b3.
00000800:00000200:3.0:1787866197.565205:0:68395:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000042229199] -> 12345-192.168.203.58@tcp (4)
00000800:00000200:3.0:1787866197.565210:0:68395:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000200:3.0:1787866197.565213:0:68395:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ea7120b3 type 3, nob 160 niov 1 nkiov 1
00000800:00000200:3.0:1787866197.565264:0:68395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1787866197.565267:0:68395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000ea7120b3.
00000400:00000200:3.0:1787866197.565271:0:68395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1787866197.565275:0:68395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: REPLY: OK
00000400:00000200:3.0:1787866197.565291:0:68395:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : REPLY - for me
00000400:00000200:3.0:1787866197.565300:0:68395:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.203.58@tcp: Reply from 12345-192.168.203.58@tcp of length 64/64 into md 0x11
00000400:00000200:3.0:1787866197.565307:0:68395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1787866197.565310:0:68395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: REPLY: OK
00000400:00000200:3.0:1787866197.565314:0:68395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000094dcefe0
00000400:00000010:3.0:1787866197.565316:0:68395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 0000000094dcefe0.
00000400:00000200:3.0:1787866197.565322:0:68395:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 3
00000400:00000200:3.0:1787866197.565325:0:68395:0:(peer.c:3119:lnet_discovery_event_reply()) Peer 192.168.203.58@tcp has discovery enabled
00000400:00000200:3.0:1787866197.565334:0:68395:0:(peer.c:3145:lnet_discovery_event_reply()) peer 192.168.203.58@tcp(0000000099ab2b98) is MR capable
00000400:00000200:3.0:1787866197.565337:0:68395:0:(peer.c:1196:lnet_peer_ni_clr_non_mr_pref_nid()) peer 192.168.203.58@tcp: 0
00000400:00000200:3.0:1787866197.565340:0:68395:0:(peer.c:3200:lnet_discovery_event_reply()) peer 192.168.203.58@tcp data present 6. state = 0x241
00000400:00000200:3.0:1787866197.565345:0:68395:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.203.58@tcp: -114
00000400:00000200:0.0:1787866197.565537:0:68386:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0
00000400:00000200:0.0:1787866197.565542:0:68386:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.203.58@tcp(0000000099ab2b98) state 0xc1
00000400:00000010:0.0:1787866197.565550:0:68386:0:(peer.c:3474:lnet_peer_merge_data()) alloc '(curnis)': 60 at 0000000036d26412.
00000400:00000010:0.0:1787866197.565553:0:68386:0:(peer.c:3475:lnet_peer_merge_data()) alloc '(addnis)': 72 at 00000000b1e206fe.
00000400:00000010:0.0:1787866197.565561:0:68386:0:(peer.c:3476:lnet_peer_merge_data()) alloc '(delnis)': 60 at 000000009cebb94f.
00000400:00000010:0.0:1787866197.565568:0:68386:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 00000000e0b98295.
00000400:00000200:0.0:1787866197.565583:0:68386:0:(peer.c:184:lnet_peer_ni_alloc()) 00000000e0b98295 nid 192.168.203.58@tcp1
00000400:00000010:0.0:1787866197.565587:0:68386:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 000000000e7381ec.
00000400:00000200:0.0:1787866197.565589:0:68386:0:(peer.c:203:lnet_peer_net_alloc()) 000000000e7381ec net tcp1
00000400:00000200:0.0:1787866197.565595:0:68386:0:(peer.c:1804:lnet_peer_attach_peer_ni()) peer 192.168.203.58@tcp NID 192.168.203.58@tcp1 flags 0x11
00000400:00000010:0.0:1787866197.565598:0:68386:0:(peer.c:3623:lnet_peer_merge_data()) kfreed 'curnis': 60 at 0000000036d26412.
00000400:00000010:0.0:1787866197.565601:0:68386:0:(peer.c:3624:lnet_peer_merge_data()) kfreed 'addnis': 72 at 00000000b1e206fe.
00000400:00000010:0.0:1787866197.565603:0:68386:0:(peer.c:3625:lnet_peer_merge_data()) kfreed 'delnis': 60 at 000000009cebb94f.
00000400:00000010:0.0:1787866197.565606:0:68386:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000129c96e9.
00000400:00000200:0.0:1787866197.565608:0:68386:0:(peer.c:3628:lnet_peer_merge_data()) peer 192.168.203.58@tcp (0000000099ab2b98): 0
00000400:00000200:0.0:1787866197.565611:0:68386:0:(peer.c:3940:lnet_peer_data_present()) peer 192.168.203.58@tcp(0000000099ab2b98): 0. state = 0x141
00000400:00000200:0.0:1787866197.565614:0:68386:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.203.58@tcp(0000000099ab2b98) state 0x141 rc 1
00000400:00000200:0.0:1787866197.565617:0:68386:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.203.58@tcp(0000000099ab2b98) state 0x141
00000400:00000010:0.0:1787866197.565621:0:68386:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d0b2e2d5.
00000400:00000010:0.0:1787866197.565628:0:68386:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000008da59ec.
00000400:00000200:0.0:1787866197.565632:0:68386:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.58@tcp
00000400:00000200:0.0:1787866197.565638:0:68386:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.203.58@tcp to MR:  192.168.203.58@tcp local destination
00000400:00000200:0.0:1787866197.565642:0:68386:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [10:10:256:4294967295] 192.168.203.58@tcp LPNI seq info [3:3:8:4294967295]
00000400:00000200:0.0:1787866197.565646:0:68386:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:0.0:1787866197.565651:0:68386:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787866197.565655:0:68386:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.203.58@tcp
00000800:00000010:0.0:1787866197.565657:0:68386:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000049f786d1.
00000800:00000200:0.0:1787866197.565661:0:68386:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000042229199] -> 12345-192.168.203.58@tcp (4)
00000800:00000200:0.0:1787866197.565665:0:68386:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000200:0.0:1787866197.565667:0:68386:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000049f786d1 type 1, nob 160 niov 1 nkiov 1
00000400:00000200:0.0:1787866197.565677:0:68386:0:(peer.c:4164:lnet_peer_send_push()) peer 192.168.203.58@tcp
00000400:00000200:0.0:1787866197.565680:0:68386:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.203.58@tcp(0000000099ab2b98) state 0x541 rc 0
00000800:00000200:0.0:1787866197.565724:0:68395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:0.0:1787866197.565726:0:68395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000049f786d1.
00000400:00000200:0.0:1787866197.565729:0:68395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1787866197.565737:0:68395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: PUT: OK
00000400:00000200:0.0:1787866197.565741:0:68395:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 5
00000400:00000200:0.0:1787866197.565743:0:68395:0:(peer.c:3241:lnet_discovery_event_send()) Push Send to 192.168.203.58@tcp: 0
00000400:00000200:0.0:1787866197.565754:0:68395:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : PUT - for me
00000400:00000200:0.0:1787866197.565760:0:68395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.58@tcp of length 64 into portal 0 MB=0x8000000000000000
00000400:00000200:0.0:1787866197.565764:0:68395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 0 from 12345-192.168.203.58@tcp of length 64/64 into md 0x5 [1] + 0
00000400:00000010:0.0:1787866197.565767:0:68395:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000fac06bd7.
00000400:00000200:0.0:1787866197.565771:0:68395:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000635f5135
00000400:00000200:0.0:1787866197.565776:0:68395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1787866197.565779:0:68395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: PUT: OK
00000400:00000200:0.0:1787866197.565781:0:68395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000635f5135
00000400:00000010:0.0:1787866197.565783:0:68395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000635f5135.
00000400:00000200:0.0:1787866197.565785:0:68395:0:(api-ni.c:2393:lnet_push_target_event_handler()) type 2 status 0 unlinked 1
00000400:00000200:0.0:1787866197.565789:0:68395:0:(peer.c:2778:lnet_peer_push_event()) peer 192.168.203.58@tcp(0000000099ab2b98) is MR
00000400:00000010:0.0:1787866197.565793:0:68395:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 73 at 00000000b1e206fe.
00000400:00000200:0.0:1787866197.565796:0:68395:0:(peer.c:2852:lnet_peer_push_event()) Received Push 192.168.203.58@tcp 6
00000400:00000200:0.0:1787866197.565798:0:68395:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.203.58@tcp: -114
00000400:00000200:0.0:1787866197.565805:0:68395:0:(lib-md.c:96:lnet_cpt_of_md()) Couldn't resolve first page of md 0000000000000000 with offset 0
00000400:00000200:0.0:1787866197.565809:0:68395:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.203.58@tcp to MR:  192.168.203.58@tcp local destination
00000400:00000200:0.0:1787866197.565812:0:68395:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [11:11:256:4294967295] 192.168.203.58@tcp LPNI seq info [4:4:8:4294967295]
00000400:00000200:0.0:1787866197.565819:0:68395:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) 192.168.203.58@tcp : ACK try# 0
00000800:00000200:0.0:1787866197.565823:0:68395:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 1 frags to 12345-192.168.203.58@tcp
00000800:00000010:0.0:1787866197.565826:0:68395:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000049f786d1.
00000800:00000200:0.0:1787866197.565829:0:68395:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000042229199] -> 12345-192.168.203.58@tcp (4)
00000800:00000200:0.0:1787866197.565848:0:68395:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000200:0.0:1787866197.565850:0:68395:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000049f786d1 type 0, nob 96 niov 1 nkiov 0
00000800:00000200:0.0:1787866197.565876:0:68395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:0.0:1787866197.565889:0:68395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000049f786d1.
00000400:00000200:0.0:1787866197.565891:0:68395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1787866197.565895:0:68395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: ACK: OK
00000400:00000200:0.0:1787866197.565920:0:68395:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : ACK - for me
00000400:00000200:0.0:1787866197.565927:0:68395:0:(lib-move.c:4539:lnet_parse_ack()) 192.168.203.58@tcp: ACK from 12345-192.168.203.58@tcp into md 0x15
00000400:00000200:0.0:1787866197.565930:0:68395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1787866197.565933:0:68395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: ACK: OK
00000400:00000200:0.0:1787866197.565935:0:68395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d0b2e2d5
00000400:00000010:0.0:1787866197.565937:0:68395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000d0b2e2d5.
00000400:00000200:0.0:1787866197.565940:0:68395:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 4
00000400:00000200:0.0:1787866197.565942:0:68395:0:(peer.c:3007:lnet_discovery_event_ack()) peer 192.168.203.58@tcp ev->status 0
00000400:00000200:0.0:1787866197.565945:0:68395:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.203.58@tcp: -114
00000400:00000200:0.0:1787866197.565956:0:68386:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0
00000400:00000010:0.0:1787866197.565960:0:68386:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000fcaafdc4.
00000400:00000010:0.0:1787866197.566764:0:68386:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d0b2e2d5.
00000400:00000200:0.0:1787866197.566773:0:68386:0:(api-ni.c:2383:lnet_push_target_post()) posted push target 0000000044c03cfa
00000400:00000200:0.0:1787866197.566778:0:68386:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.203.58@tcp(0000000099ab2b98) state 0xc1
00000400:00000010:0.0:1787866197.566784:0:68386:0:(peer.c:3474:lnet_peer_merge_data()) alloc '(curnis)': 60 at 000000009cebb94f.
00000400:00000010:0.0:1787866197.566786:0:68386:0:(peer.c:3475:lnet_peer_merge_data()) alloc '(addnis)': 72 at 000000004a4d9ba9.
00000400:00000010:0.0:1787866197.566788:0:68386:0:(peer.c:3476:lnet_peer_merge_data()) alloc '(delnis)': 60 at 0000000036d26412.
00000400:00000010:0.0:1787866197.566792:0:68386:0:(peer.c:3623:lnet_peer_merge_data()) kfreed 'curnis': 60 at 000000009cebb94f.
00000400:00000010:0.0:1787866197.566794:0:68386:0:(peer.c:3624:lnet_peer_merge_data()) kfreed 'addnis': 72 at 000000004a4d9ba9.
00000400:00000010:0.0:1787866197.566796:0:68386:0:(peer.c:3625:lnet_peer_merge_data()) kfreed 'delnis': 60 at 0000000036d26412.
00000400:00000010:0.0:1787866197.566799:0:68386:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 73 at 00000000b1e206fe.
00000400:00000200:0.0:1787866197.566801:0:68386:0:(peer.c:3628:lnet_peer_merge_data()) peer 192.168.203.58@tcp (0000000099ab2b98): 0
00000400:00000200:0.0:1787866197.566804:0:68386:0:(peer.c:3940:lnet_peer_data_present()) peer 192.168.203.58@tcp(0000000099ab2b98): 0. state = 0x141
00000400:00000200:0.0:1787866197.566807:0:68386:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.203.58@tcp(0000000099ab2b98) state 0x141 rc 1
00000400:00000200:0.0:1787866197.566810:0:68386:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.203.58@tcp(0000000099ab2b98) state 0x141
00000400:00000200:0.0:1787866197.566812:0:68386:0:(peer.c:4082:lnet_peer_discovered()) peer 192.168.203.58@tcp
00000400:00000200:0.0:1787866197.566814:0:68386:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.203.58@tcp(0000000099ab2b98) state 0x111 rc 0
00000400:00000200:0.0:1787866197.566817:0:68386:0:(peer.c:2629:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 192.168.203.58@tcp
00000400:00000200:3.0:1787866197.566941:0:68636:0:(peer.c:2984:lnet_discover_peer_locked()) peer 192.168.203.58@tcp NID 192.168.203.58@tcp: 0. discovery complete
00000800:00000010:1.0:1787866197.579377:0:68390:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 00000000a83d358f.
00000800:00000010:1.0:1787866197.579398:0:68390:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 0000000062b5bb53.
00000800:00000010:1.0:1787866197.579406:0:68390:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000970d892d.
00000400:00000200:2.0:1787866197.580154:0:68398:0:(acceptor.c:578:lnet_acceptor()) Accepted connection on ens2:988
00000400:00000200:2.0:1787866197.580187:0:68398:0:(acceptor.c:502:lnet_accept()) Accept 192.168.203.58@tcp from 254.3.0.2
00000800:00000010:2.0:1787866197.580191:0:68398:0:(socklnd.c:737:ksocknal_accept()) alloc '(cr)': 32 at 000000006c7f0227.
00000001:02000400:0.0:1787866197.580617:0:68637:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 0
00000800:00000010:3.0:1787866197.581972:0:68392:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 00000000c7c98440.
00000800:00000010:3.0:1787866197.581984:0:68392:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 00000000c8236af5.
00000800:00000010:1.0:1787866197.584169:0:68390:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000970d892d.
00000800:00000010:3.0:1787866197.588758:0:68392:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 000000007fb542a5.
00000800:00000200:3.1F:1787866197.588775:0:68392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000042229199] -> 12345-192.168.203.58@tcp (4)
00000800:00000200:3.1:1787866197.588788:0:68392:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.203.58@tcp 000000007fb542a5 deleted
00000800:00000010:3.1:1787866197.588790:0:68392:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 000000007fb542a5.
00000800:00000200:3.1:1787866197.588795:0:68392:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 3, ksnr_connected a ksnr_max_conns 1
00000800:00000200:3.0:1787866197.588802:0:68392:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.203.58@tcp p 3.x 192.168.203.58 -> 192.168.203.58:1022 incarnation:1787866195749305484 sched[0]
00000800:00000010:3.0:1787866197.588808:0:68392:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000bd7350f0.
00000800:00000200:1.1:1787866197.589286:0:68390:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 2, ksnr_connected e ksnr_max_conns 1
00000800:00000200:1.0:1787866197.589294:0:68390:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.203.58@tcp p 3.x 192.168.203.58 -> 192.168.203.58:988 incarnation:1787866195749305484 sched[0]
00000800:00000010:1.0:1787866197.589298:0:68390:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 0000000062b5bb53.
00000800:00000010:3.0:1787866197.589666:0:68392:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000bd7350f0.
00000800:00000010:3.0:1787866197.589671:0:68392:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 00000000c8236af5.
00000800:00000010:3.0:1787866197.589693:0:68392:0:(socklnd_cb.c:2179:ksocknal_connd()) kfreed 'cr': 32 at 000000006c7f0227.
00000400:00000010:3.0:1787866197.633672:0:68640:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 12 at 000000005b9b1bb4.
00000400:00000010:3.0:1787866197.633685:0:68640:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 12 at 000000005b9b1bb4.
00000001:02000400:0.0:1787866197.659417:0:68642:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set max_recovery_ping_interval 4
00000001:02000400:1.0:1787866197.710190:0:68645:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp -d *@tcp -m GET -r 1 -e local_error
00000400:00000200:2.0:1787866197.753624:0:68646:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866197.753633:0:68646:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866197.753645:0:68646:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp
00000400:00000200:2.0:1787866197.753647:0:68646:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866197.753650:0:68646:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp
00000400:00000200:2.0:1787866197.753651:0:68646:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866197.753655:0:68646:0:(api-ni.c:3207:nla_strnid()) <?> -> <?>
00000400:00000200:2.0:1787866197.753656:0:68646:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866197.753658:0:68646:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866197.753659:0:68646:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866197.753661:0:68646:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866197.753662:0:68646:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866197.753664:0:68646:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866197.753666:0:68646:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000010:2.0:1787866197.753670:0:68646:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 000000005f6d7a2c.
00000400:00000200:2.0:1787866197.753677:0:68646:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp, dst 255.255.255.255@tcp, rate 1, interval 0
00000001:02000400:0.0:1787866197.767688:0:68647:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp1 -d *@tcp1 -m GET -r 1 -e local_error
00000400:00000200:1.0:1787866197.834681:0:68648:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866197.834690:0:68648:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866197.834702:0:68648:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1
00000400:00000200:1.0:1787866197.834704:0:68648:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866197.834707:0:68648:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1
00000400:00000200:1.0:1787866197.834709:0:68648:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866197.834711:0:68648:0:(api-ni.c:3207:nla_strnid()) <?> -> <?>
00000400:00000200:1.0:1787866197.834712:0:68648:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866197.834713:0:68648:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866197.834714:0:68648:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866197.834716:0:68648:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866197.834717:0:68648:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866197.834724:0:68648:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866197.834726:0:68648:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000010:1.0:1787866197.834730:0:68648:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 00000000a9f9657c.
00000400:00000200:1.0:1787866197.834738:0:68648:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp1, dst 255.255.255.255@tcp1, rate 1, interval 0
00000001:02000400:0.0:1787866197.865115:0:68649:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp -d *@tcp -r 1
00000400:00000200:1.0:1787866197.906243:0:68650:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866197.906252:0:68650:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866197.906263:0:68650:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp
00000400:00000200:1.0:1787866197.906264:0:68650:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866197.906267:0:68650:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp
00000400:00000200:1.0:1787866197.906268:0:68650:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866197.906271:0:68650:0:(api-ni.c:3207:nla_strnid()) <?> -> <?>
00000400:00000200:1.0:1787866197.906272:0:68650:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866197.906273:0:68650:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866197.906275:0:68650:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866197.906276:0:68650:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866197.906278:0:68650:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866197.906280:0:68650:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866197.906281:0:68650:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000010:1.0:1787866197.906285:0:68650:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 0000000009e622a7.
00000400:00000200:1.0:1787866197.906293:0:68650:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp, dst 255.255.255.255@tcp, rate 1, interval 0
00000001:02000400:1.0:1787866197.921646:0:68651:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp1 -d *@tcp1 -r 1
00000400:00000200:2.0:1787866197.951399:0:68652:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866197.951408:0:68652:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866197.951421:0:68652:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1
00000400:00000200:2.0:1787866197.951423:0:68652:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866197.951425:0:68652:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1
00000400:00000200:2.0:1787866197.951427:0:68652:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866197.951429:0:68652:0:(api-ni.c:3207:nla_strnid()) <?> -> <?>
00000400:00000200:2.0:1787866197.951430:0:68652:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866197.951431:0:68652:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866197.951433:0:68652:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866197.951434:0:68652:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866197.951435:0:68652:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866197.951436:0:68652:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:2.0:1787866197.951438:0:68652:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000010:2.0:1787866197.951443:0:68652:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 000000003983c4eb.
00000400:00000200:2.0:1787866197.951450:0:68652:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp1, dst 255.255.255.255@tcp1, rate 1, interval 0
00000001:02000400:0.0:1787866197.966524:0:68653:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net set --health 0 --nid 192.168.203.58@tcp
00000400:00000010:2.0:1787866198.012057:0:68654:0:(api-ni.c:6877:lnet_net_cmd()) alloc '(conf)': 808 at 00000000cd707e0b.
00000400:00000010:2.0:1787866198.012079:0:68654:0:(api-ni.c:6555:lnet_genl_parse_local_ni()) alloc '(tun)': 172 at 00000000c08bf30d.
00000400:00000200:2.0:1787866198.012083:0:68654:0:(api-ni.c:6621:lnet_genl_parse_local_ni()) Requested NID 192.168.203.58@tcp
00000400:00020000:2.0:1787866198.012096:0:68654:0:(api-ni.c:4473:lnet_ni_set_healthv()) manually adding local NI 192.168.203.58@tcp to recovery
00000400:00000200:2.0:1787866198.026240:0:68654:0:(lib-lnet.h:200:lnet_ni_set_status_locked()) ni 192.168.203.58@tcp status changed from 0x15aac0de to 0xdeadface
00000400:00000010:2.0:1787866198.026262:0:68654:0:(api-ni.c:6852:lnet_genl_parse_local_ni()) kfreed 'tun': 172 at 00000000c08bf30d.
00000400:00000010:2.0:1787866198.026270:0:68654:0:(api-ni.c:6994:lnet_net_cmd()) kfreed 'conf': 808 at 00000000cd707e0b.
00000400:00000010:3.0:1787866198.083266:0:68660:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000004ab50932.
00000400:00000010:3.0:1787866198.083372:0:68660:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000004ab50932.
00000400:00000200:2.0:1787866198.741391:0:68385:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.203.58@tcp
00000400:00000010:2.0:1787866198.741402:0:68385:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000003164c3ac.
00000400:00000010:2.0:1787866198.741411:0:68385:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000002a39adf6.
00000400:00000010:2.0:1787866198.741436:0:68385:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007692f314.
00000400:00000010:2.0:1787866198.741457:0:68385:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000925783b6.
00000400:00000200:2.0:1787866198.741462:0:68385:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.203.58@tcp
00000400:00000200:2.0:1787866198.741474:0:68385:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR:  192.168.203.58@tcp local destination
00000400:00000200:2.0:1787866198.741480:0:68385:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.58@tcp
00000400:00000200:2.0:1787866198.741484:0:68385:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [12:12:256:4294967295] 192.168.203.58@tcp LPNI seq info [5:5:8:4294967295]
00000400:00000200:2.0:1787866198.741491:0:68385:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:2.0:1787866198.741498:0:68385:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:<?>) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : GET try# 0
00000800:00000200:2.0:1787866198.741503:0:68385:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.203.58@tcp
00000800:00000010:2.0:1787866198.741508:0:68385:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000000e3043f0.
00000800:00000200:2.0:1787866198.741515:0:68385:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000042229199] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:2.0:1787866198.741524:0:68385:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000200:2.0:1787866198.741527:0:68385:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000000e3043f0 type 2, nob 96 niov 1 nkiov 0
00000400:00000200:2.0:1787866198.741566:0:68385:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.203.58@tcp added to recovery queue. ping count: 1 next ping: 1410 health :0
00000400:00000200:0.0:1787866198.745768:0:68396:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 1
00000400:00000200:0.0:1787866198.745778:0:68396:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.203.58@tcp(192.168.203.58@tcp)->dst 192.168.203.58@tcp: GET simulate health error: LOCAL_ERROR
00000800:02000400:0.0:1787866198.745790:0:68396:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.203.58: rc = -22
00000800:00000200:0.0:1787866198.763479:0:68396:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [00000000751322ba] Error -22 on write to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000010:0.0:1787866198.763488:0:68396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000000e3043f0.
00000400:00000200:0.0:1787866198.763496:0:68396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 5
00000400:00000200:0.0:1787866198.763501:0:68396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: GET: LOCAL_ERROR
00000400:00000200:0.0:1787866198.763512:0:68396:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5
00000400:00000200:0.0:1787866198.763515:0:68396:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.203.58@tcp recovery message sent unsuccessfully:-5
00000400:00000200:0.0:1787866198.763518:0:68396:0:(lib-move.c:4071:lnet_handle_recovery_reply()) local NI (192.168.203.58@tcp) recovery failed with -5
00000400:00000010:0.0:1787866199.184778:0:68670:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000f2fdfffa.
00000400:00000010:0.0:1787866199.184908:0:68670:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000f2fdfffa.
00000400:00000200:2.0:1787866199.763008:0:68385:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007692f314
00000400:00000010:2.0:1787866199.763029:0:68385:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000007692f314.
00000400:00000200:2.0:1787866199.763037:0:68385:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0
00000400:00000200:2.0:1787866199.763043:0:68385:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.203.58@tcp recovery ping unlinked
00000400:00000010:2.0:1787866199.763046:0:68385:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000003164c3ac.
00000400:00000010:2.0:1787866199.763051:0:68385:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000002a39adf6.
00000400:00000010:3.0:1787866200.268744:0:68680:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000004ab50932.
00000400:00000010:3.0:1787866200.270295:0:68680:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000004ab50932.
00000400:00000200:2.0:1787866200.787094:0:68385:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.203.58@tcp
00000400:00000010:2.0:1787866200.787104:0:68385:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000008c8a7b83.
00000400:00000010:2.0:1787866200.787112:0:68385:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000002b7c957a.
00000400:00000010:2.0:1787866200.787117:0:68385:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007692f314.
00000400:00000010:2.0:1787866200.787126:0:68385:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000448ce6df.
00000400:00000200:2.0:1787866200.787130:0:68385:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.203.58@tcp
00000400:00000200:2.0:1787866200.787157:0:68385:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR:  192.168.203.58@tcp local destination
00000400:00000200:2.0:1787866200.787163:0:68385:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.58@tcp
00000400:00000200:2.0:1787866200.787167:0:68385:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [13:13:256:4294967295] 192.168.203.58@tcp LPNI seq info [6:6:8:4294967295]
00000400:00000200:2.0:1787866200.787173:0:68385:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:2.0:1787866200.787179:0:68385:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:<?>) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : GET try# 0
00000800:00000200:2.0:1787866200.787185:0:68385:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.203.58@tcp
00000800:00000010:2.0:1787866200.787189:0:68385:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000003f012825.
00000800:00000200:2.0:1787866200.787196:0:68385:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000042229199] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:2.0:1787866200.787213:0:68385:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000200:2.0:1787866200.787217:0:68385:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003f012825 type 2, nob 96 niov 1 nkiov 0
00000400:00000200:2.0:1787866200.787243:0:68385:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.203.58@tcp added to recovery queue. ping count: 2 next ping: 1414 health :0
00000400:00000200:0.0:1787866200.787389:0:68395:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 2
00000400:00000200:0.0:1787866200.787395:0:68395:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.203.58@tcp(192.168.203.58@tcp)->dst 192.168.203.58@tcp: GET simulate health error: LOCAL_ERROR
00000800:02000400:0.0:1787866200.787398:0:68395:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.203.58: rc = -22
00000800:00000200:0.0:1787866200.787406:0:68395:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [000000008d8cdc37] Error -22 on write to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000010:0.0:1787866200.787410:0:68395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000003f012825.
00000400:00000200:0.0:1787866200.787416:0:68395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 5
00000400:00000200:0.0:1787866200.787420:0:68395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: GET: LOCAL_ERROR
00000400:00000200:0.0:1787866200.787425:0:68395:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5
00000400:00000200:0.0:1787866200.787428:0:68395:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.203.58@tcp recovery message sent unsuccessfully:-5
00000400:00000200:0.0:1787866200.787432:0:68395:0:(lib-move.c:4071:lnet_handle_recovery_reply()) local NI (192.168.203.58@tcp) recovery failed with -5
00000400:00000010:0.0:1787866201.419746:0:68691:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000f2fdfffa.
00000400:00000010:0.0:1787866201.419884:0:68691:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000f2fdfffa.
00000400:00000010:2.0:1787866201.465091:0:68695:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000402763fd.
00000400:00000010:2.0:1787866201.465510:0:68695:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000402763fd.
00000400:00000200:0.0:1787866201.502670:0:68702:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 0
00000400:00000010:0.0:1787866201.502683:0:68702:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 0000000027dd214c.
00000400:00000200:0.0:1787866201.502693:0:68702:0:(api-ni.c:9567:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:0.0:1787866201.502709:0:68702:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 000000002c484b56.
00000400:00000200:0.0:1787866201.502716:0:68702:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:0.0:1787866201.502720:0:68702:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 000000002c484b56.
00000400:00000010:0.0:1787866201.502728:0:68702:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 0000000027dd214c.
00000001:02000400:2.0:1787866201.519518:0:68704:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -l
00000400:00000200:1.0:1787866201.533239:0:68705:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 0
00000400:00000010:1.0:1787866201.533248:0:68705:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000001dbdf0da.
00000400:00000200:1.0:1787866201.533257:0:68705:0:(api-ni.c:9567:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:1.0:1787866201.533274:0:68705:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 0000000096088a15.
00000400:00000200:1.0:1787866201.533281:0:68705:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:1.0:1787866201.533286:0:68705:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 0000000096088a15.
00000400:00000010:1.0:1787866201.533292:0:68705:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000001dbdf0da.
00000400:00000010:1.0:1787866201.580273:0:68711:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000001f6718ce.
00000400:00000010:1.0:1787866201.580399:0:68711:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000001f6718ce.
00000400:00000200:2.0:1787866201.811243:0:68385:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007692f314
00000400:00000010:2.0:1787866201.811253:0:68385:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000007692f314.
00000400:00000200:2.0:1787866201.811260:0:68385:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0
00000400:00000200:2.0:1787866201.811267:0:68385:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.203.58@tcp recovery ping unlinked
00000400:00000010:2.0:1787866201.811271:0:68385:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000008c8a7b83.
00000400:00000010:2.0:1787866201.811276:0:68385:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000002b7c957a.
00000400:00000010:1.0:1787866202.655184:0:68721:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000001f6718ce.
00000400:00000010:1.0:1787866202.655378:0:68721:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000001f6718ce.
00000400:00000010:2.0:1787866203.743811:0:68731:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000402763fd.
00000400:00000010:2.0:1787866203.743932:0:68731:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000402763fd.
00000400:00000200:2.0:1787866204.883527:0:68385:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.203.58@tcp
00000400:00000010:2.0:1787866204.883536:0:68385:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000035a1ff94.
00000400:00000010:2.0:1787866204.883548:0:68385:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000002eacacef.
00000400:00000010:2.0:1787866204.883554:0:68385:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007692f314.
00000400:00000010:2.0:1787866204.883564:0:68385:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a639457c.
00000400:00000200:2.0:1787866204.883569:0:68385:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.203.58@tcp
00000400:00000200:2.0:1787866204.883586:0:68385:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR:  192.168.203.58@tcp local destination
00000400:00000200:2.0:1787866204.883595:0:68385:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.58@tcp
00000400:00000200:2.0:1787866204.883599:0:68385:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [14:14:256:4294967295] 192.168.203.58@tcp LPNI seq info [7:7:8:4294967295]
00000400:00000200:2.0:1787866204.883614:0:68385:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:2.0:1787866204.883621:0:68385:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:<?>) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : GET try# 0
00000800:00000200:2.0:1787866204.883627:0:68385:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.203.58@tcp
00000800:00000010:2.0:1787866204.883633:0:68385:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000045b5ed8.
00000800:00000200:2.0:1787866204.883640:0:68385:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000042229199] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:2.0:1787866204.883650:0:68385:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000200:2.0:1787866204.883655:0:68385:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000045b5ed8 type 2, nob 96 niov 1 nkiov 0
00000400:00000200:2.0:1787866204.883671:0:68385:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.203.58@tcp added to recovery queue. ping count: 3 next ping: 1418 health :0
00000400:00000200:2.0:1787866204.883699:0:68397:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 3
00000400:00000200:2.0:1787866204.883705:0:68397:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.203.58@tcp(192.168.203.58@tcp)->dst 192.168.203.58@tcp: GET simulate health error: LOCAL_ERROR
00000800:02000400:2.0:1787866204.883709:0:68397:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.203.58: rc = -22
00000400:00000010:1.0:1787866204.883738:0:68741:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000001f6718ce.
00000400:00000010:1.0:1787866204.883865:0:68741:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000001f6718ce.
00000800:00000200:2.0:1787866204.917276:0:68397:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [00000000751322ba] Error -22 on write to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000010:2.0:1787866204.917293:0:68397:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000045b5ed8.
00000400:00000200:2.0:1787866204.917300:0:68397:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 5
00000400:00000200:2.0:1787866204.917306:0:68397:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: GET: LOCAL_ERROR
00000400:00000200:2.0:1787866204.917319:0:68397:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5
00000400:00000200:2.0:1787866204.917323:0:68397:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.203.58@tcp recovery message sent unsuccessfully:-5
00000400:00000200:2.0:1787866204.917327:0:68397:0:(lib-move.c:4071:lnet_handle_recovery_reply()) local NI (192.168.203.58@tcp) recovery failed with -5
00000400:00000200:0.0:1787866205.907161:0:68385:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007692f314
00000400:00000010:0.0:1787866205.907181:0:68385:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000007692f314.
00000400:00000200:0.0:1787866205.907192:0:68385:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0
00000400:00000200:0.0:1787866205.907200:0:68385:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.203.58@tcp recovery ping unlinked
00000400:00000010:0.0:1787866205.907204:0:68385:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000035a1ff94.
00000400:00000010:0.0:1787866205.907211:0:68385:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000002eacacef.
00000400:00000010:0.0:1787866205.996616:0:68751:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000f2fdfffa.
00000400:00000010:0.0:1787866205.997380:0:68751:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000f2fdfffa.
00000400:00000010:3.0:1787866207.128288:0:68762:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000004ab50932.
00000400:00000010:3.0:1787866207.128440:0:68762:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000004ab50932.
00000400:00000010:1.0:1787866208.248772:0:68772:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000001f6718ce.
00000400:00000010:1.0:1787866208.248911:0:68772:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000001f6718ce.
00000400:00000200:0.0:1787866208.979274:0:68385:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.203.58@tcp
00000400:00000010:0.0:1787866208.979285:0:68385:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000b6d5ead9.
00000400:00000010:0.0:1787866208.979295:0:68385:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000025f50f78.
00000400:00000010:0.0:1787866208.979302:0:68385:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000300d3ee4.
00000400:00000010:0.0:1787866208.979322:0:68385:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000093c30b01.
00000400:00000200:0.0:1787866208.979327:0:68385:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.203.58@tcp
00000400:00000200:0.0:1787866208.979340:0:68385:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR:  192.168.203.58@tcp local destination
00000400:00000200:0.0:1787866208.979346:0:68385:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.58@tcp
00000400:00000200:0.0:1787866208.979350:0:68385:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [15:15:256:4294967295] 192.168.203.58@tcp LPNI seq info [8:8:8:4294967295]
00000400:00000200:0.0:1787866208.979366:0:68385:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:0.0:1787866208.979372:0:68385:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:<?>) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : GET try# 0
00000800:00000200:0.0:1787866208.979377:0:68385:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.203.58@tcp
00000800:00000010:0.0:1787866208.979383:0:68385:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000049f786d1.
00000800:00000200:0.0:1787866208.979389:0:68385:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000042229199] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:0.0:1787866208.979404:0:68385:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000200:0.0:1787866208.979408:0:68385:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000049f786d1 type 2, nob 96 niov 1 nkiov 0
00000400:00000200:0.0:1787866208.979432:0:68385:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.203.58@tcp added to recovery queue. ping count: 4 next ping: 1422 health :0
00000400:00000200:2.0:1787866208.979485:0:68396:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 4
00000400:00000200:2.0:1787866208.979493:0:68396:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.203.58@tcp(192.168.203.58@tcp)->dst 192.168.203.58@tcp: GET simulate health error: LOCAL_ERROR
00000800:02000400:2.0:1787866208.979496:0:68396:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.203.58: rc = -22
00000800:00000200:2.0:1787866208.979501:0:68396:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [000000008d8cdc37] Error -22 on write to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000010:2.0:1787866208.979506:0:68396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000049f786d1.
00000400:00000200:2.0:1787866208.979513:0:68396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 5
00000400:00000200:2.0:1787866208.979517:0:68396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: GET: LOCAL_ERROR
00000400:00000200:2.0:1787866208.979522:0:68396:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5
00000400:00000200:2.0:1787866208.979525:0:68396:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.203.58@tcp recovery message sent unsuccessfully:-5
00000400:00000200:2.0:1787866208.979535:0:68396:0:(lib-move.c:4071:lnet_handle_recovery_reply()) local NI (192.168.203.58@tcp) recovery failed with -5
00000400:00000010:2.0:1787866209.354107:0:68782:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000402763fd.
00000400:00000010:2.0:1787866209.354217:0:68782:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000402763fd.
00000400:00000010:3.0:1787866209.392222:0:68786:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000004ab50932.
00000400:00000010:3.0:1787866209.392342:0:68786:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000004ab50932.
00000400:00000200:3.0:1787866209.448268:0:68793:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 0
00000400:00000010:3.0:1787866209.448276:0:68793:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 0000000040c0507c.
00000400:00000200:3.0:1787866209.448286:0:68793:0:(api-ni.c:9567:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:3.0:1787866209.448318:0:68793:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 0000000020709b6a.
00000400:00000200:3.0:1787866209.448327:0:68793:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:3.0:1787866209.448331:0:68793:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 0000000020709b6a.
00000400:00000010:3.0:1787866209.448339:0:68793:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 0000000040c0507c.
00000001:02000400:0.0:1787866209.474146:0:68795:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -l
00000400:00000200:0.0:1787866209.490774:0:68796:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 0
00000400:00000010:0.0:1787866209.490782:0:68796:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000bf4957f7.
00000400:00000200:0.0:1787866209.490794:0:68796:0:(api-ni.c:9567:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:0.0:1787866209.490824:0:68796:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000561a9f90.
00000400:00000200:0.0:1787866209.490863:0:68796:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:0.0:1787866209.490868:0:68796:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000561a9f90.
00000400:00000010:0.0:1787866209.490877:0:68796:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000bf4957f7.
00000001:02000400:1.0:1787866209.504166:0:68797:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net_drop del -a
00000400:00000200:0.0:1787866209.517393:0:68798:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866209.517402:0:68798:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866209.517407:0:68798:0:(api-ni.c:3207:nla_strnid()) <?> -> <?>
00000400:00000200:0.0:1787866209.517408:0:68798:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866209.517410:0:68798:0:(api-ni.c:3207:nla_strnid()) <?> -> <?>
00000400:00000200:0.0:1787866209.517413:0:68798:0:(net_fault.c:254:lnet_drop_rule_del()) src <?> dst <?>
00000400:00000200:0.0:1787866209.517421:0:68798:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp->dst: 255.255.255.255@tcp (1/1, 0)
00000400:00000010:0.0:1787866209.517423:0:68798:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 000000005f6d7a2c.
00000400:00000200:0.0:1787866209.517430:0:68798:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp1->dst: 255.255.255.255@tcp1 (1/1, 0)
00000400:00000010:0.0:1787866209.517431:0:68798:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 00000000a9f9657c.
00000400:00000200:0.0:1787866209.517434:0:68798:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp->dst: 255.255.255.255@tcp (1/1, 0)
00000400:00000010:0.0:1787866209.517436:0:68798:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 0000000009e622a7.
00000400:00000200:0.0:1787866209.517440:0:68798:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp1->dst: 255.255.255.255@tcp1 (1/1, 0)
00000400:00000010:0.0:1787866209.517441:0:68798:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 000000003983c4eb.
00000001:02000400:0.0:1787866209.530140:0:68799:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set max_recovery_ping_interval 900
00000400:00000200:0.0:1787866210.002909:0:68385:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000300d3ee4
00000400:00000010:0.0:1787866210.002919:0:68385:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000300d3ee4.
00000400:00000200:0.0:1787866210.002927:0:68385:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0
00000400:00000200:0.0:1787866210.002933:0:68385:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.203.58@tcp recovery ping unlinked
00000400:00000010:0.0:1787866210.002937:0:68385:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000b6d5ead9.
00000400:00000010:0.0:1787866210.002941:0:68385:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000025f50f78.
00000400:00000200:2.0:1787866213.074912:0:68385:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.203.58@tcp
00000400:00000010:2.0:1787866213.074918:0:68385:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000bad506c6.
00000400:00000010:2.0:1787866213.074927:0:68385:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000cff26568.
00000400:00000010:2.0:1787866213.074932:0:68385:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f4dd8e9c.
00000400:00000010:2.0:1787866213.074941:0:68385:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000f637a5be.
00000400:00000200:2.0:1787866213.074946:0:68385:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.203.58@tcp
00000400:00000200:2.0:1787866213.074958:0:68385:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR:  192.168.203.58@tcp local destination
00000400:00000200:2.0:1787866213.074965:0:68385:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.58@tcp
00000400:00000200:2.0:1787866213.074969:0:68385:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [16:16:256:4294967295] 192.168.203.58@tcp LPNI seq info [9:9:8:4294967295]
00000400:00000200:2.0:1787866213.074975:0:68385:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:2.0:1787866213.074981:0:68385:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:<?>) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : GET try# 0
00000800:00000200:2.0:1787866213.074986:0:68385:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.203.58@tcp
00000800:00000010:2.0:1787866213.074991:0:68385:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000045b5ed8.
00000800:00000200:2.0:1787866213.074997:0:68385:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000042229199] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:2.0:1787866213.075005:0:68385:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000200:2.0:1787866213.075009:0:68385:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000045b5ed8 type 2, nob 96 niov 1 nkiov 0
00000400:00000200:2.0:1787866213.075035:0:68385:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.203.58@tcp added to recovery queue. ping count: 5 next ping: 1454 health :0
00000800:00000200:1.0:1787866213.075646:0:68395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1787866213.075652:0:68395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000045b5ed8.
00000400:00000200:1.0:1787866213.075657:0:68395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1787866213.075662:0:68395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: GET: OK
00000400:00000200:1.0:1787866213.075667:0:68395:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0
00000400:00000200:1.0:1787866213.075669:0:68395:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.203.58@tcp recovery message sent successfully:0
00000400:00000200:1.0:1787866213.075695:0:68395:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : GET - for me
00000400:00000200:1.0:1787866213.075717:0:68395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.58@tcp of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:1.0:1787866213.075724:0:68395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.203.58@tcp of length 64/272 into md 0xd [1] + 0
00000400:00000200:1.0:1787866213.075733:0:68395:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.203.58@tcp to MR:  192.168.203.58@tcp local destination
00000400:00000200:1.0:1787866213.075736:0:68395:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [17:17:256:4294967295] 192.168.203.58@tcp LPNI seq info [10:10:8:4294967295]
00000400:00000200:1.0:1787866213.075743:0:68395:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) 192.168.203.58@tcp : REPLY try# 0
00000800:00000200:1.0:1787866213.075747:0:68395:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.203.58@tcp
00000800:00000010:1.0:1787866213.075751:0:68395:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000009dab4a5c.
00000800:00000200:1.0:1787866213.075755:0:68395:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000042229199] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:1.0:1787866213.075759:0:68395:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000200:1.0:1787866213.075762:0:68395:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000009dab4a5c type 3, nob 160 niov 1 nkiov 1
00000800:00000200:1.0:1787866213.075804:0:68395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1787866213.075806:0:68395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000009dab4a5c.
00000400:00000200:1.0:1787866213.075809:0:68395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1787866213.075812:0:68395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: REPLY: OK
00000400:00000200:1.0:1787866213.075827:0:68395:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : REPLY - for me
00000400:00000200:1.0:1787866213.075859:0:68395:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.203.58@tcp: Reply from 12345-192.168.203.58@tcp of length 64/64 into md 0x2d
00000400:00000200:1.0:1787866213.075867:0:68395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1787866213.075870:0:68395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: REPLY: OK
00000400:00000200:1.0:1787866213.075874:0:68395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f4dd8e9c
00000400:00000010:1.0:1787866213.075876:0:68395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000f4dd8e9c.
00000400:00000200:1.0:1787866213.075879:0:68395:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0
00000400:00000010:1.0:1787866213.075881:0:68395:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000bad506c6.
00000400:00000010:1.0:1787866213.075891:0:68395:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000cff26568.
00000001:02000400:0.0:1787866216.582743:0:69356:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 211: Remote NI recovery checks ======= 17:30:15 (1787866215)
00000001:02000400:0.0:1787866216.918873:0:69370:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
00000400:00000200:3.0:1787866216.946532:0:69371:0:(net_fault.c:254:lnet_drop_rule_del()) src <?> dst <?>
00000400:00000200:2.0:1787866216.947314:0:68386:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: -108
00000400:00000200:2.0:1787866216.947319:0:68386:0:(peer.c:4385:lnet_peer_discovery()) stopping
00000400:00000200:2.0:1787866216.947334:0:68386:0:(peer.c:4431:lnet_peer_discovery()) stopped
00000400:00000200:3.0:1787866216.947932:0:69371:0:(peer.c:4488:lnet_peer_discovery_stop()) discovery stopped
00000400:00000010:3.0:1787866216.953414:0:69371:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 00000000833e52df.
00000400:00000010:3.0:1787866216.953427:0:69371:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 0000000008da59ec.
00000400:00000010:3.0:1787866216.953430:0:69371:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 00000000925783b6.
00000400:00000010:3.0:1787866216.953432:0:69371:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 00000000448ce6df.
00000400:00000010:3.0:1787866216.953434:0:69371:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 00000000a639457c.
00000400:00000010:3.0:1787866216.953436:0:69371:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 0000000093c30b01.
00000400:00000010:3.0:1787866216.953437:0:69371:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 00000000f637a5be.
00000400:00000010:3.0:1787866216.953440:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000b840e06f.
00000400:00000010:3.0:1787866216.953443:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000005c5b9a21.
00000400:00000010:3.0:1787866216.953451:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000055791137.
00000400:00000010:3.0:1787866216.953453:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000001cdd4679.
00000400:00000010:3.0:1787866216.953460:0:69371:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000fcaafdc4.
00000400:00000200:3.0:1787866216.953464:0:69371:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d0b2e2d5
00000400:00000010:3.0:1787866216.953466:0:69371:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000d0b2e2d5.
00000400:00000200:3.0:1787866216.953471:0:69371:0:(api-ni.c:2393:lnet_push_target_event_handler()) type 6 status 0 unlinked 1
00000400:00000010:3.0:1787866216.953474:0:69371:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000044c03cfa.
00000400:00000400:3.0:1787866216.953488:0:69371:0:(lib-ptl.c:964:lnet_clear_lazy_portal()) Active lazy portal 0 on exit
00000400:00000010:3.0:1787866216.956869:0:69371:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000001cbfee5.
00000400:00000200:3.0:1787866216.956883:0:69371:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ece06eaf
00000400:00000010:3.0:1787866216.956886:0:69371:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000ece06eaf.
00000400:00000010:3.0:1787866216.956895:0:69371:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 73 at 000000007148613d.
00000400:00000200:1.0:1787866216.957028:0:68398:0:(acceptor.c:659:lnet_acceptor()) Acceptor stopping
00000400:00000200:3.0:1787866216.960425:0:69371:0:(router.c:792:lnet_del_route()) Del route: net <255:65535> : gw <?>
00000400:00000200:3.0:1787866216.960453:0:69371:0:(lo.c:49:lolnd_shutdown()) shutdown
00000400:00000010:3.0:1787866216.960456:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000c06af4da.
00000400:00000010:3.0:1787866216.960462:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000009ac881c6.
00000400:00000010:3.0:1787866216.960465:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000bd77456a.
00000400:00000010:3.0:1787866216.960466:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000006e65badb.
00000400:00000010:3.0:1787866216.960469:0:69371:0:(config.c:293:lnet_ni_free()) kfreed 'ni': 552 at 00000000c3a2a8bc.
00000400:00000010:3.0:1787866216.960479:0:69371:0:(config.c:329:lnet_net_free()) kfreed 'net': 176 at 00000000a1110acd.
00000800:00000200:3.1:1787866216.960493:0:69371:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 2, ksnr_connected a ksnr_max_conns 1
00000800:00000200:3.1:1787866216.960557:0:69371:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 3, ksnr_connected 2 ksnr_max_conns 1
00000800:00000200:3.1:1787866216.960589:0:69371:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 1, ksnr_connected 2 ksnr_max_conns 1
00000800:00000200:3.1:1787866216.960621:0:69371:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 1, ksnr_connected 0 ksnr_max_conns 1
00000800:00000010:3.1:1787866216.960624:0:69371:0:(socklnd.c:126:ksocknal_destroy_conn_cb()) kfreed 'conn_cb': 192 at 000000000be8e223.
00000400:00020000:3.0:1787866216.960631:0:69371:0:(acceptor.c:252:lnet_acceptor_remove_socket()) Interface ens2 not found
00000800:00000200:2.0:1787866216.963288:0:68394:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 00000000a83d358f
00000800:00000010:2.0:1787866216.963298:0:68394:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 00000000a83d358f.
00000800:00000200:2.0:1787866216.963303:0:68394:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 00000000751322ba
00000800:00000010:2.0:1787866216.963304:0:68394:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 00000000751322ba.
00000800:00000200:2.0:1787866216.963407:0:68394:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 00000000c7c98440
00000800:00000010:2.0:1787866216.963410:0:68394:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 00000000c7c98440.
00000800:00000200:2.0:1787866216.963416:0:68394:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 000000008d8cdc37
00000800:00000200:2.0:1787866216.963424:0:68394:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.203.58@tcp 0000000042229199 deleted
00000800:00000010:2.0:1787866216.963426:0:68394:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 0000000042229199.
00000800:00000010:2.0:1787866216.963440:0:68394:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 000000008d8cdc37.
00000800:00000010:3.0:1787866216.966623:0:69371:0:(socklnd.c:2498:ksocknal_shutdown()) kfreed 'net': 208 at 00000000ada1255e.
00000400:02000000:3.0:1787866216.966645:0:69371:0:(api-ni.c:2563:lnet_clear_zombies_nis_locked()) Removed LNI 192.168.203.58@tcp
00000400:00000010:3.0:1787866216.983319:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000001d65a08f.
00000400:00000010:3.0:1787866216.983331:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000006110df10.
00000400:00000010:3.0:1787866216.983333:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000bd940ac1.
00000400:00000010:3.0:1787866216.983335:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000f84b69db.
00000400:00000010:3.0:1787866216.983354:0:69371:0:(config.c:286:lnet_ni_free()) kfreed 'ni->ni_interface': 5 at 0000000066b316e0.
00000400:00000010:3.0:1787866216.983357:0:69371:0:(config.c:293:lnet_ni_free()) kfreed 'ni': 552 at 0000000087d0447c.
00000400:00000200:3.0:1787866216.983374:0:69371:0:(peer.c:620:lnet_peer_ni_clr_local_net_locked()) 00000000cb8e99f4 nid 192.168.203.58@tcp detached from local net
00000400:00000010:2.0:1787866216.983876:0:69371:0:(config.c:329:lnet_net_free()) kfreed 'net': 176 at 00000000d8ae2599.
00000400:00020000:2.0:1787866216.983903:0:69371:0:(acceptor.c:252:lnet_acceptor_remove_socket()) Interface ens2 not found
00000800:00000010:2.0:1787866216.983906:0:69371:0:(socklnd.c:2498:ksocknal_shutdown()) kfreed 'net': 208 at 000000005d82d75f.
00000800:00000010:2.0:1787866216.983910:0:69371:0:(socklnd.c:2235:ksocknal_base_shutdown()) before NAL cleanup: kmem 418789
00000800:00000200:2.0:1787866216.983917:0:69371:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_GOING_DOWN, iface=lo ifindex 1 state 0 ns 00000000198b8817
00000800:00000200:2.0:1787866216.983921:0:69371:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_DOWN, iface=lo ifindex 1 state 0 ns 00000000198b8817
00000800:00000200:2.0:1787866216.983923:0:69371:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UNREGISTER, iface=lo ifindex 1 state 0 ns 00000000198b8817
00000800:00000200:2.0:1787866216.983925:0:69371:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_GOING_DOWN, iface=ens2 ifindex 2 state 6 ns 00000000198b8817
00000800:00000200:2.0:1787866216.983927:0:69371:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_DOWN, iface=ens2 ifindex 2 state 6 ns 00000000198b8817
00000800:00000200:2.0:1787866216.983929:0:69371:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UNREGISTER, iface=ens2 ifindex 2 state 6 ns 00000000198b8817
00000400:00000010:2.0:1787866216.997338:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 000000004eef61b3.
00000400:00000010:2.0:1787866216.997348:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000062c8fb52.
00000800:00000010:2.0:1787866216.997352:0:69371:0:(socklnd.c:2296:ksocknal_base_shutdown()) after NAL cleanup: kmem 418637
00000400:02000000:2.0:1787866216.997361:0:69371:0:(api-ni.c:2563:lnet_clear_zombies_nis_locked()) Removed LNI 192.168.203.58@tcp1
00000400:00000010:2.0:1787866217.005325:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000009c0bc3d7.
00000400:00000010:2.0:1787866217.005339:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000b27d37fd.
00000400:00000010:2.0:1787866217.005342:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000002b9ed566.
00000400:00000010:2.0:1787866217.005344:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000004742348e.
00000400:00000010:2.0:1787866217.005346:0:69371:0:(config.c:286:lnet_ni_free()) kfreed 'ni->ni_interface': 5 at 000000002ea4affc.
00000400:00000010:2.0:1787866217.005350:0:69371:0:(config.c:293:lnet_ni_free()) kfreed 'ni': 552 at 00000000ddcffa3a.
00000400:00000200:2.0:1787866217.005366:0:69371:0:(peer.c:442:lnet_peer_del_locked()) peer 192.168.203.58@tcp1
00000400:00000200:2.0:1787866217.005371:0:69371:0:(peer.c:364:lnet_peer_detach_peer_ni_locked()) peer 192.168.203.58@tcp1 NID 192.168.203.58@tcp
00000400:00000200:2.0:1787866217.005374:0:69371:0:(peer.c:2392:lnet_destroy_peer_ni_locked()) 00000000cb8e99f4 nid 192.168.203.58@tcp
00000400:00000010:2.0:1787866217.005377:0:69371:0:(peer.c:2422:lnet_destroy_peer_ni_locked()) kfreed 'lpni': 480 at 00000000cb8e99f4.
00000400:00000200:2.0:1787866217.005380:0:69371:0:(peer.c:213:lnet_destroy_peer_net_locked()) 0000000018fba111 net tcp
00000400:00000010:2.0:1787866217.005382:0:69371:0:(peer.c:220:lnet_destroy_peer_net_locked()) kfreed 'lpn': 72 at 0000000018fba111.
00000400:00000200:2.0:1787866217.005386:0:69371:0:(peer.c:364:lnet_peer_detach_peer_ni_locked()) peer 192.168.203.58@tcp1 NID 192.168.203.58@tcp1
00000400:00000200:2.0:1787866217.005388:0:69371:0:(peer.c:2392:lnet_destroy_peer_ni_locked()) 00000000e0b98295 nid 192.168.203.58@tcp1
00000400:00000010:2.0:1787866217.005389:0:69371:0:(peer.c:2422:lnet_destroy_peer_ni_locked()) kfreed 'lpni': 480 at 00000000e0b98295.
00000400:00000200:2.0:1787866217.005392:0:69371:0:(peer.c:213:lnet_destroy_peer_net_locked()) 000000000e7381ec net tcp1
00000400:00000010:2.0:1787866217.005411:0:69371:0:(peer.c:220:lnet_destroy_peer_net_locked()) kfreed 'lpn': 72 at 000000000e7381ec.
00000400:00000200:2.0:1787866217.005415:0:69371:0:(peer.c:274:lnet_destroy_peer_locked()) 0000000099ab2b98 nid 192.168.203.58@tcp1
00000400:00000010:2.0:1787866217.005417:0:69371:0:(peer.c:304:lnet_destroy_peer_locked()) kfreed 'lp': 360 at 0000000099ab2b98.
00000400:00000010:2.0:1787866217.005424:0:69371:0:(config.c:329:lnet_net_free()) kfreed 'net': 176 at 00000000942431d4.
00000400:00000010:2.0:1787866217.005430:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000738b1d36.
00000400:00000010:2.0:1787866217.005432:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000e3073140.
00000400:00000010:2.0:1787866217.005438:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000005083cb8a.
00000400:00000010:2.0:1787866217.005443:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000be3d3d27.
00000400:00000010:2.0:1787866217.005445:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000e7f6ed49.
00000400:00000010:2.0:1787866217.005447:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000b27f351f.
00000400:00000010:2.0:1787866217.005450:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f29e49df.
00000400:00000010:2.0:1787866217.005453:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000007aed3052.
00000400:00000010:2.0:1787866217.005455:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000000c3faf23.
00000400:00000010:2.0:1787866217.005457:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e6757c30.
00000400:00000010:2.0:1787866217.005460:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000063414f75.
00000400:00000010:2.0:1787866217.005462:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000dfd99f4c.
00000400:00000010:2.0:1787866217.005464:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000586537e0.
00000400:00000010:2.0:1787866217.005465:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000000f2b7a98.
00000400:00000010:2.0:1787866217.005469:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000088e78596.
00000400:00000010:2.0:1787866217.005471:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f9cb8dba.
00000400:00000010:2.0:1787866217.005473:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000005c3c6f51.
00000400:00000010:2.0:1787866217.005475:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000005e1ca33c.
00000400:00000010:2.0:1787866217.005478:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000079ea313f.
00000400:00000010:2.0:1787866217.005480:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000006ca9eb8.
00000400:00000010:2.0:1787866217.005482:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000003507272.
00000400:00000010:2.0:1787866217.005483:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000010a6e38f.
00000400:00000010:2.0:1787866217.005486:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f8e9a5a4.
00000400:00000010:2.0:1787866217.005489:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000874a2026.
00000400:00000010:2.0:1787866217.005490:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000741d98d1.
00000400:00000010:2.0:1787866217.005492:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000019d8f1d6.
00000400:00000010:2.0:1787866217.005495:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000e4747cb8.
00000400:00000010:2.0:1787866217.005497:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000003f0bdf4c.
00000400:00000010:2.0:1787866217.005499:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000a85b7985.
00000400:00000010:2.0:1787866217.005509:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000009dea2467.
00000400:00000010:2.0:1787866217.005513:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000000ee9133.
00000400:00000010:2.0:1787866217.005518:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000aab07357.
00000400:00000010:2.0:1787866217.005520:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000b8416bbd.
00000400:00000010:2.0:1787866217.005521:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000002d74340f.
00000400:00000010:2.0:1787866217.005524:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000003506a16b.
00000400:00000010:2.0:1787866217.005528:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000000a23be36.
00000400:00000010:2.0:1787866217.005530:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000060ac5e87.
00000400:00000010:2.0:1787866217.005531:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000ae718185.
00000400:00000010:2.0:1787866217.005534:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000000b3665f6.
00000400:00000010:2.0:1787866217.005537:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000001d9cf5b7.
00000400:00000010:2.0:1787866217.005538:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000000b98e9f.
00000400:00000010:2.0:1787866217.005540:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000c2ec0b30.
00000400:00000010:2.0:1787866217.005543:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000eff1234f.
00000400:00000010:2.0:1787866217.005545:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000004d9a224a.
00000400:00000010:2.0:1787866217.005547:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000d74fa375.
00000400:00000010:2.0:1787866217.005549:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000088e7fc31.
00000400:00000010:2.0:1787866217.005552:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f35d978d.
00000400:00000010:2.0:1787866217.005556:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000015c73426.
00000400:00000010:2.0:1787866217.005557:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000007213af63.
00000400:00000010:2.0:1787866217.005559:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000aa387aa5.
00000400:00000010:2.0:1787866217.005563:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000091bd4b17.
00000400:00000010:2.0:1787866217.005565:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000639234ad.
00000400:00000010:2.0:1787866217.005566:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000c58a0ec5.
00000400:00000010:2.0:1787866217.005568:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000094312786.
00000400:00000010:2.0:1787866217.005571:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000006e575a31.
00000400:00000010:2.0:1787866217.005574:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000001a01547f.
00000400:00000010:2.0:1787866217.005576:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000005d1d0d9f.
00000400:00000010:2.0:1787866217.005578:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000007bf2fb4c.
00000400:00000010:2.0:1787866217.005581:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ed353615.
00000400:00000010:2.0:1787866217.005583:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000003c4758a4.
00000400:00000010:2.0:1787866217.005585:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000083988c41.
00000400:00000010:2.0:1787866217.005586:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000013205a57.
00000400:00000010:2.0:1787866217.005596:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000874db7d2.
00000400:00000010:2.0:1787866217.005598:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000037f27f8a.
00000400:00000010:2.0:1787866217.005600:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000097374f24.
00000400:00000010:2.0:1787866217.005602:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000012c6c579.
00000400:00000010:2.0:1787866217.005605:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ede22331.
00000400:00000010:2.0:1787866217.005606:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000012f17a2c.
00000400:00000010:2.0:1787866217.005608:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000175db05b.
00000400:00000010:2.0:1787866217.005609:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000375896ff.
00000400:00000010:2.0:1787866217.005612:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000042dece22.
00000400:00000010:2.0:1787866217.005613:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000002df42b09.
00000400:00000010:2.0:1787866217.005615:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000fcd82b23.
00000400:00000010:2.0:1787866217.005617:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000036b3a50e.
00000400:00000010:2.0:1787866217.005620:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000dbb65273.
00000400:00000010:2.0:1787866217.005622:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000008647e19c.
00000400:00000010:2.0:1787866217.005623:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000f439e6df.
00000400:00000010:2.0:1787866217.005625:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000048d6c5d0.
00000400:00000010:2.0:1787866217.005628:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000b355e253.
00000400:00000010:2.0:1787866217.005631:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000d42ef1be.
00000400:00000010:2.0:1787866217.005632:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000442aa5e4.
00000400:00000010:2.0:1787866217.005634:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000be2546b6.
00000400:00000010:2.0:1787866217.005637:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ff0fa99f.
00000400:00000010:2.0:1787866217.005639:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000667cff50.
00000400:00000010:2.0:1787866217.005641:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000001f9bae5c.
00000400:00000010:2.0:1787866217.005642:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000007ae10c44.
00000400:00000010:2.0:1787866217.005645:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000003cb04dca.
00000400:00000010:2.0:1787866217.005648:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000e4bb642e.
00000400:00000010:2.0:1787866217.005650:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000000874e15d.
00000400:00000010:2.0:1787866217.005651:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000aaa9c056.
00000400:00000010:2.0:1787866217.005654:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ad1ad778.
00000400:00000010:2.0:1787866217.005656:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000006c810cfd.
00000400:00000010:2.0:1787866217.005658:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000004d802733.
00000400:00000010:2.0:1787866217.005660:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000fd0ac38c.
00000400:00000010:2.0:1787866217.005663:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000001fc9d250.
00000400:00000010:2.0:1787866217.005670:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000005978b3b9.
00000400:00000010:2.0:1787866217.005672:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000170eda37.
00000400:00000010:2.0:1787866217.005673:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000ab15aaf2.
00000400:00000010:2.0:1787866217.005676:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000066fd5643.
00000400:00000010:2.0:1787866217.005678:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000095ccba5d.
00000400:00000010:2.0:1787866217.005679:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000007e493f7e.
00000400:00000010:2.0:1787866217.005681:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000a4918ca0.
00000400:00000010:2.0:1787866217.005684:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000372d9216.
00000400:00000010:2.0:1787866217.005686:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f579002a.
00000400:00000010:2.0:1787866217.005687:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000e1fe24dc.
00000400:00000010:2.0:1787866217.005689:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000072029be1.
00000400:00000010:2.0:1787866217.005692:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000af0841f6.
00000400:00000010:2.0:1787866217.005694:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000002a40b4c5.
00000400:00000010:2.0:1787866217.005696:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000007d9f1444.
00000400:00000010:2.0:1787866217.005698:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000072de1828.
00000400:00000010:2.0:1787866217.005700:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000eec38694.
00000400:00000010:2.0:1787866217.005703:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000e399c7d7.
00000400:00000010:2.0:1787866217.005704:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000040339714.
00000400:00000010:2.0:1787866217.005706:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000a3257d5d.
00000400:00000010:2.0:1787866217.005709:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000007654107.
00000400:00000010:2.0:1787866217.005711:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000060bf75e9.
00000400:00000010:2.0:1787866217.005713:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000006c8b3f32.
00000400:00000010:2.0:1787866217.005714:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000070635d03.
00000400:00000010:2.0:1787866217.005717:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000007666a5df.
00000400:00000010:2.0:1787866217.005719:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000005ee7d883.
00000400:00000010:2.0:1787866217.005720:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000e143976c.
00000400:00000010:2.0:1787866217.005722:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000016ffbbec.
00000400:00000010:2.0:1787866217.005725:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000004da4d315.
00000400:00000010:2.0:1787866217.005727:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000006f7734d5.
00000400:00000010:2.0:1787866217.005729:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000006bf21ed9.
00000400:00000010:2.0:1787866217.005730:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000008787625a.
00000400:00000010:2.0:1787866217.005734:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000005604167f.
00000400:00000010:2.0:1787866217.005736:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000002d1b0c79.
00000400:00000010:2.0:1787866217.005742:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002d0d725f.
00000400:00000010:2.0:1787866217.005744:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000009ffd5ad9.
00000400:00000010:2.0:1787866217.005747:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000028b9089e.
00000400:00000010:2.0:1787866217.005762:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000000c2cb86.
00000400:00000010:2.0:1787866217.005764:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000e20bfbc2.
00000400:00000010:2.0:1787866217.005766:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000012d03674.
00000400:00000010:2.0:1787866217.005769:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000049f79d76.
00000400:00000010:2.0:1787866217.005771:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000066728e94.
00000400:00000010:2.0:1787866217.005773:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000005eb9e1b7.
00000400:00000010:2.0:1787866217.005774:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000003ad4337c.
00000400:00000010:2.0:1787866217.005778:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000002f504b20.
00000400:00000010:2.0:1787866217.005780:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000dfde04a9.
00000400:00000010:2.0:1787866217.005782:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000cf73c735.
00000400:00000010:2.0:1787866217.005783:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000b2a3b6a8.
00000400:00000010:2.0:1787866217.005786:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f3b8bfe7.
00000400:00000010:2.0:1787866217.005788:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000087d0ea8e.
00000400:00000010:2.0:1787866217.005789:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000086cf63a3.
00000400:00000010:2.0:1787866217.005791:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e4649c08.
00000400:00000010:2.0:1787866217.005794:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000066cc6be4.
00000400:00000010:2.0:1787866217.005796:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000006efe2553.
00000400:00000010:2.0:1787866217.005798:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000e0038e5b.
00000400:00000010:2.0:1787866217.005799:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e9590d32.
00000400:00000010:2.0:1787866217.005802:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000d94bb90a.
00000400:00000010:2.0:1787866217.005804:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000907726e0.
00000400:00000010:2.0:1787866217.005806:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000d4a008b6.
00000400:00000010:2.0:1787866217.005807:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000938cb033.
00000400:00000010:2.0:1787866217.005810:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ff320cb3.
00000400:00000010:2.0:1787866217.005812:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f6653601.
00000400:00000010:2.0:1787866217.005814:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000000861544c.
00000400:00000010:2.0:1787866217.005815:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000cc56ef7a.
00000400:00000010:2.0:1787866217.005819:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000010775746.
00000400:00000010:2.0:1787866217.005821:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000ca7f81c8.
00000400:00000010:2.0:1787866217.005823:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000053233d68.
00000400:00000010:2.0:1787866217.005862:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000ab4cbdf5.
00000400:00000010:2.0:1787866217.005867:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000006fe7da78.
00000400:00000010:2.0:1787866217.005869:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000039e568ed.
00000400:00000010:2.0:1787866217.005871:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000971cb4da.
00000400:00000010:2.0:1787866217.005873:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000007da5d9b4.
00000400:00000010:2.0:1787866217.005875:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f68aaa16.
00000400:00000010:2.0:1787866217.005877:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000075b49a8d.
00000400:00000010:2.0:1787866217.005879:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000c4ab0945.
00000400:00000010:2.0:1787866217.005881:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000a8e3b738.
00000400:00000010:2.0:1787866217.005884:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000006ce923da.
00000400:00000010:2.0:1787866217.005898:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000b90a4287.
00000400:00000010:2.0:1787866217.005901:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000005736ae64.
00000400:00000010:2.0:1787866217.005903:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000b0bcd302.
00000400:00000010:2.0:1787866217.005906:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000067e117b8.
00000400:00000010:2.0:1787866217.005908:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000002d0f2d30.
00000400:00000010:2.0:1787866217.005909:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000003dc95ae.
00000400:00000010:2.0:1787866217.005911:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000b3612087.
00000400:00000010:2.0:1787866217.005914:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000003dca47e3.
00000400:00000010:2.0:1787866217.005916:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000d9b6c465.
00000400:00000010:2.0:1787866217.005918:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000004b6bbae8.
00000400:00000010:2.0:1787866217.005919:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000056efd22.
00000400:00000010:2.0:1787866217.005922:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000052006a4e.
00000400:00000010:2.0:1787866217.005924:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f95cda80.
00000400:00000010:2.0:1787866217.005925:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000069363a92.
00000400:00000010:2.0:1787866217.005927:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000021f0813b.
00000400:00000010:2.0:1787866217.005930:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000a3e62e9d.
00000400:00000010:2.0:1787866217.005932:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000095220e4a.
00000400:00000010:2.0:1787866217.005933:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000d8a71836.
00000400:00000010:2.0:1787866217.005935:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000040d72397.
00000400:00000010:2.0:1787866217.005938:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000799598b3.
00000400:00000010:2.0:1787866217.005940:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000054555ad8.
00000400:00000010:2.0:1787866217.005942:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000345411b6.
00000400:00000010:2.0:1787866217.005943:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000ca528c1f.
00000400:00000010:2.0:1787866217.005952:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000e53a5eee.
00000400:00000010:2.0:1787866217.005954:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000027b2bdf8.
00000400:00000010:2.0:1787866217.005956:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000ce64a59b.
00000400:00000010:2.0:1787866217.005957:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000093c9fcf8.
00000400:00000010:2.0:1787866217.005960:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000bce067fc.
00000400:00000010:2.0:1787866217.005962:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000005b139ed2.
00000400:00000010:2.0:1787866217.005963:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000c9a4fd72.
00000400:00000010:2.0:1787866217.005965:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000007545239e.
00000400:00000010:2.0:1787866217.005967:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000006d892fb3.
00000400:00000010:2.0:1787866217.005969:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000021031acc.
00000400:00000010:2.0:1787866217.005971:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000943f16cf.
00000400:00000010:2.0:1787866217.005972:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000f615b3f7.
00000400:00000010:2.0:1787866217.005975:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000072a8f16f.
00000400:00000010:2.0:1787866217.005976:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000e80ae833.
00000400:00000010:2.0:1787866217.005978:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000001c3ad77c.
00000400:00000010:2.0:1787866217.005980:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000001a570e11.
00000400:00000010:2.0:1787866217.005982:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000076d6cd1c.
00000400:00000010:2.0:1787866217.005984:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000d28068b0.
00000400:00000010:2.0:1787866217.005986:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000065590823.
00000400:00000010:2.0:1787866217.005987:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000003ac074f2.
00000400:00000010:2.0:1787866217.005990:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000015c7a150.
00000400:00000010:2.0:1787866217.005992:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000007c81eb09.
00000400:00000010:2.0:1787866217.005994:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000006af87bde.
00000400:00000010:2.0:1787866217.005995:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000024d8becd.
00000400:00000010:2.0:1787866217.005998:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000007434b255.
00000400:00000010:2.0:1787866217.006013:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000d7a60416.
00000400:00000010:2.0:1787866217.006015:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000008876f365.
00000400:00000010:2.0:1787866217.006018:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000007154a7e2.
00000400:00000010:2.0:1787866217.006021:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000cb480af2.
00000400:00000010:2.0:1787866217.006022:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000055425607.
00000400:00000010:2.0:1787866217.006024:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000023cd7e93.
00000400:00000010:2.0:1787866217.006025:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e3cbe3fc.
00000400:00000010:2.0:1787866217.006028:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000a27db30f.
00000400:00000010:2.0:1787866217.006036:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000aa4f13b0.
00000400:00000010:2.0:1787866217.006038:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000ef2cd2ea.
00000400:00000010:2.0:1787866217.006040:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000a5aed5b6.
00000400:00000010:2.0:1787866217.006043:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ce65c75a.
00000400:00000010:2.0:1787866217.006045:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000a17cc9d9.
00000400:00000010:2.0:1787866217.006046:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000005586a200.
00000400:00000010:2.0:1787866217.006048:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000008d8322ba.
00000400:00000010:2.0:1787866217.006050:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000a01446bf.
00000400:00000010:2.0:1787866217.006052:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000083953ed1.
00000400:00000010:2.0:1787866217.006054:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000001e54cbd7.
00000400:00000010:2.0:1787866217.006056:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000009f0e7ba3.
00000400:00000010:2.0:1787866217.006059:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000061615e9.
00000400:00000010:2.0:1787866217.006061:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000e3093529.
00000400:00000010:2.0:1787866217.006063:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000026f6c9c0.
00000400:00000010:2.0:1787866217.006065:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e0f4b49a.
00000400:00000010:2.0:1787866217.006068:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000002bcb5613.
00000400:00000010:2.0:1787866217.006070:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000030c6bb9b.
00000400:00000010:2.0:1787866217.006071:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000009467422c.
00000400:00000010:2.0:1787866217.006073:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000086becfe9.
00000400:00000010:2.0:1787866217.006076:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ec41b373.
00000400:00000010:2.0:1787866217.006079:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000005608000.
00000400:00000010:2.0:1787866217.006080:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000cab59421.
00000400:00000010:2.0:1787866217.006082:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000c46746d3.
00000400:00000010:2.0:1787866217.006085:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000005d29569c.
00000400:00000010:2.0:1787866217.006092:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000c4efb49e.
00000400:00000010:2.0:1787866217.006094:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000004dc43276.
00000400:00000010:2.0:1787866217.006095:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000b4b34503.
00000400:00000010:2.0:1787866217.006098:0:69371:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000085e71ae7.
00000400:00000010:2.0:1787866217.006100:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000001af34ebb.
00000400:00000010:2.0:1787866217.006102:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000de4d9430.
00000400:00000010:2.0:1787866217.006103:0:69371:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000023ff55ad.
00000400:00000010:2.0:1787866217.006105:0:69371:0:(lib-ptl.c:854:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals': 512 at 000000007c65fb77.
00000400:00000010:2.0:1787866217.006108:0:69371:0:(api-ni.c:1205:lnet_res_container_cleanup()) kfreed 'rec->rec_lh_hash': 65536 at 00000000b0147152.
00000400:00000010:2.0:1787866217.006128:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f9a8cb83.
00000400:00000010:2.0:1787866217.006131:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000ea68bc5b.
00000400:00000010:2.0:1787866217.006132:0:69371:0:(api-ni.c:1205:lnet_res_container_cleanup()) kfreed 'rec->rec_lh_hash': 65536 at 0000000083486692.
00000400:00000010:2.0:1787866217.006136:0:69371:0:(lib-msg.c:1308:lnet_msg_container_cleanup()) kfreed 'container->msc_finalizers': 32 at 00000000fccb4c56.
00000400:00000010:2.0:1787866217.006138:0:69371:0:(lib-msg.c:1314:lnet_msg_container_cleanup()) kfreed 'container->msc_resenders': 32 at 00000000f2e30eec.
00000400:00000010:2.0:1787866217.006140:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 00000000aa17dfbf.
00000400:00000010:2.0:1787866217.006142:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000062c72d1a.
00000400:00000010:2.0:1787866217.006147:0:69371:0:(peer.c:85:lnet_peer_tables_destroy()) kfreed 'hash': 8192 at 000000008198bbcd.
00000400:00000010:2.0:1787866217.006156:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 00000000b23e87d0.
00000400:00000010:2.0:1787866217.006158:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000006ef8a308.
00000400:00000010:2.0:1787866217.006160:0:69371:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 00000000cffd9ef1.
00000400:00000010:2.0:1787866217.006162:0:69371:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000cd87e7a3.
00000400:00000010:2.0:1787866217.006165:0:69371:0:(api-ni.c:700:lnet_destroy_remote_nets_table()) kfreed 'the_lnet.ln_remote_nets_hash': 1024 at 00000000d21aed04.
00000001:00000200:2.0:1787866217.006168:0:69371:0:(udsp.c:1126:lnet_udsp_destroy()) Destroying UDSPs in the system
00000001:02000400:0.0:1787866217.033566:0:69372:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure
00000400:00000010:2.0:1787866217.064750:0:69373:0:(api-ni.c:676:lnet_create_remote_nets_table()) alloc '(hash)': 1024 at 00000000550bd6db.
00000400:00000010:2.0:1787866217.064758:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000007a722ddd.
00000400:00000010:2.0:1787866217.064761:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000e68bbf54.
00000400:00000010:2.0:1787866217.064763:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000038e32123.
00000400:00000010:2.0:1787866217.064764:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 000000004a143be2.
00000400:00000010:2.0:1787866217.064768:0:69373:0:(peer.c:109:lnet_peer_tables_create()) alloc '(hash)': 8192 at 00000000a721e894.
00000400:00000010:2.0:1787866217.064771:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000035a1ff94.
00000400:00000010:2.0:1787866217.064772:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000c0dd528f.
00000400:00000010:2.0:1787866217.064774:0:69373:0:(lib-msg.c:1338:lnet_msg_container_setup()) alloc '(container->msc_finalizers)': 32 at 000000002b7fb679.
00000400:00000010:2.0:1787866217.064776:0:69373:0:(lib-msg.c:1348:lnet_msg_container_setup()) alloc '(container->msc_resenders)': 32 at 0000000066cecd94.
00000400:00000010:2.0:1787866217.064974:0:69373:0:(api-ni.c:1227:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 00000000479850c5.
00000400:00000010:2.0:1787866217.064992:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000030a803c4.
00000400:00000010:2.0:1787866217.064995:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009ad4c87e.
00000400:00000010:2.0:1787866217.065100:0:69373:0:(api-ni.c:1227:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 000000001fd70cf7.
00000400:00000010:2.0:1787866217.065119:0:69373:0:(lib-ptl.c:864:lnet_portals_create()) alloc '(the_lnet.ln_portals)': 512 at 00000000149ca4aa.
00000400:00000010:2.0:1787866217.065123:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000050cc6800.
00000400:00000010:2.0:1787866217.065125:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000eb1b95d7.
00000400:00000010:2.0:1787866217.065128:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000007071892.
00000400:00000010:2.0:1787866217.065138:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000066b6d98f.
00000400:00000010:2.0:1787866217.065141:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000317fbbc1.
00000400:00000010:2.0:1787866217.065143:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000028069879.
00000400:00000010:2.0:1787866217.065145:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000280e6db6.
00000400:00000010:2.0:1787866217.065149:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f783d8b3.
00000400:00000010:2.0:1787866217.065151:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000067e51d17.
00000400:00000010:2.0:1787866217.065153:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000543069cf.
00000400:00000010:2.0:1787866217.065155:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000bed862ec.
00000400:00000010:2.0:1787866217.065159:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b1b59a22.
00000400:00000010:2.0:1787866217.065161:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000000770e695.
00000400:00000010:2.0:1787866217.065163:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000007cbed98b.
00000400:00000010:2.0:1787866217.065165:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f3dfce7b.
00000400:00000010:2.0:1787866217.065169:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ec41b373.
00000400:00000010:2.0:1787866217.065172:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000005aa09076.
00000400:00000010:2.0:1787866217.065174:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002262e8a9.
00000400:00000010:2.0:1787866217.065175:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006fab2734.
00000400:00000010:2.0:1787866217.065179:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002bcb5613.
00000400:00000010:2.0:1787866217.065181:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000c5f9be91.
00000400:00000010:2.0:1787866217.065183:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000006bdf237.
00000400:00000010:2.0:1787866217.065185:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003872e290.
00000400:00000010:2.0:1787866217.065188:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000061615e9.
00000400:00000010:2.0:1787866217.065191:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000077764818.
00000400:00000010:2.0:1787866217.065192:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000060a6f9d.
00000400:00000010:2.0:1787866217.065194:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a6a5a280.
00000400:00000010:2.0:1787866217.065198:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a01446bf.
00000400:00000010:2.0:1787866217.065200:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000edc9be73.
00000400:00000010:2.0:1787866217.065202:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000006c7f0227.
00000400:00000010:2.0:1787866217.065204:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f48cfbf4.
00000400:00000010:2.0:1787866217.065207:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ce65c75a.
00000400:00000010:2.0:1787866217.065209:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000006e79233e.
00000400:00000010:2.0:1787866217.065211:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e945666a.
00000400:00000010:2.0:1787866217.065213:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000038e79002.
00000400:00000010:2.0:1787866217.065222:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a27db30f.
00000400:00000010:2.0:1787866217.065225:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000557c67ca.
00000400:00000010:2.0:1787866217.065227:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000003164c3ac.
00000400:00000010:2.0:1787866217.065229:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000efa8010d.
00000400:00000010:2.0:1787866217.065232:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000cb480af2.
00000400:00000010:2.0:1787866217.065235:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000014e18e51.
00000400:00000010:2.0:1787866217.065237:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000004c8208e0.
00000400:00000010:2.0:1787866217.065238:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000215b7ab4.
00000400:00000010:2.0:1787866217.065242:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007434b255.
00000400:00000010:2.0:1787866217.065245:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000c980c965.
00000400:00000010:2.0:1787866217.065246:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001c04d9f8.
00000400:00000010:2.0:1787866217.065248:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000082f13997.
00000400:00000010:2.0:1787866217.065264:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000eff1234f.
00000400:00000010:2.0:1787866217.065267:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000005408f50.
00000400:00000010:2.0:1787866217.065269:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001923fd6d.
00000400:00000010:2.0:1787866217.065271:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c9c5a6f3.
00000400:00000010:2.0:1787866217.065275:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000000b3665f6.
00000400:00000010:2.0:1787866217.065277:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000001b28f015.
00000400:00000010:2.0:1787866217.065279:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002673251f.
00000400:00000010:2.0:1787866217.065280:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000476ae7b8.
00000400:00000010:2.0:1787866217.065284:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003506a16b.
00000400:00000010:2.0:1787866217.065287:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000060507bb8.
00000400:00000010:2.0:1787866217.065288:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e621c0b1.
00000400:00000010:2.0:1787866217.065290:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000cefc814e.
00000400:00000010:2.0:1787866217.065294:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000000ee9133.
00000400:00000010:2.0:1787866217.065296:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000003ffeab57.
00000400:00000010:2.0:1787866217.065297:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000891e59fa.
00000400:00000010:2.0:1787866217.065299:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000082484639.
00000400:00000010:2.0:1787866217.065303:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e4747cb8.
00000400:00000010:2.0:1787866217.065305:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000f0f413bc.
00000400:00000010:2.0:1787866217.065306:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000c43a83d3.
00000400:00000010:2.0:1787866217.065307:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000024bb7037.
00000400:00000010:2.0:1787866217.065316:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f8e9a5a4.
00000400:00000010:2.0:1787866217.065319:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000008428b227.
00000400:00000010:2.0:1787866217.065320:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ebf511a2.
00000400:00000010:2.0:1787866217.065322:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003a15ec44.
00000400:00000010:2.0:1787866217.065325:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000079ea313f.
00000400:00000010:2.0:1787866217.065327:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000003b33edc5.
00000400:00000010:2.0:1787866217.065328:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000b44b6829.
00000400:00000010:2.0:1787866217.065330:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000aafd4e56.
00000400:00000010:2.0:1787866217.065334:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000088e78596.
00000400:00000010:2.0:1787866217.065336:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000008ccd18f3.
00000400:00000010:2.0:1787866217.065338:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000003d983f9f.
00000400:00000010:2.0:1787866217.065340:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000006c75b56.
00000400:00000010:2.0:1787866217.065344:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ad1ad778.
00000400:00000010:2.0:1787866217.065347:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e4494135.
00000400:00000010:2.0:1787866217.065348:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000c6b11cde.
00000400:00000010:2.0:1787866217.065350:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004d5d243f.
00000400:00000010:2.0:1787866217.065353:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003cb04dca.
00000400:00000010:2.0:1787866217.065355:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000003aea9b.
00000400:00000010:2.0:1787866217.065357:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000fa88b919.
00000400:00000010:2.0:1787866217.065358:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000033b0a5ba.
00000400:00000010:2.0:1787866217.065362:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ff0fa99f.
00000400:00000010:2.0:1787866217.065365:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000384aa030.
00000400:00000010:2.0:1787866217.065367:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000004c7e3611.
00000400:00000010:2.0:1787866217.065369:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fb3bbee8.
00000400:00000010:2.0:1787866217.065372:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b355e253.
00000400:00000010:2.0:1787866217.065374:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000dc4a8d7c.
00000400:00000010:2.0:1787866217.065376:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000c33ae974.
00000400:00000010:2.0:1787866217.065377:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fed419e0.
00000400:00000010:2.0:1787866217.065382:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000dbb65273.
00000400:00000010:2.0:1787866217.065384:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000057596936.
00000400:00000010:2.0:1787866217.065385:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000feccc3d4.
00000400:00000010:2.0:1787866217.065387:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007620a0d3.
00000400:00000010:2.0:1787866217.065395:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000042dece22.
00000400:00000010:2.0:1787866217.065398:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000d98881dd.
00000400:00000010:2.0:1787866217.065399:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000cf1de98d.
00000400:00000010:2.0:1787866217.065401:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d72b201a.
00000400:00000010:2.0:1787866217.065404:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ede22331.
00000400:00000010:2.0:1787866217.065406:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000072c42fde.
00000400:00000010:2.0:1787866217.065408:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000068ee1672.
00000400:00000010:2.0:1787866217.065409:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001630e016.
00000400:00000010:2.0:1787866217.065412:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000874db7d2.
00000400:00000010:2.0:1787866217.065414:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000069f2c576.
00000400:00000010:2.0:1787866217.065416:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000f123ceb5.
00000400:00000010:2.0:1787866217.065417:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000036c04c13.
00000400:00000010:2.0:1787866217.065422:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ed353615.
00000400:00000010:2.0:1787866217.065424:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000001ce22a9f.
00000400:00000010:2.0:1787866217.065425:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000086763905.
00000400:00000010:2.0:1787866217.065427:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002e3843d1.
00000400:00000010:2.0:1787866217.065430:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006e575a31.
00000400:00000010:2.0:1787866217.065432:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000009087bf54.
00000400:00000010:2.0:1787866217.065433:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000f2893b01.
00000400:00000010:2.0:1787866217.065434:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000093c4a148.
00000400:00000010:2.0:1787866217.065437:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000091bd4b17.
00000400:00000010:2.0:1787866217.065439:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000003dd6ae70.
00000400:00000010:2.0:1787866217.065440:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000093d22730.
00000400:00000010:2.0:1787866217.065442:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000b579dd1.
00000400:00000010:2.0:1787866217.065445:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f35d978d.
00000400:00000010:2.0:1787866217.065447:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e96b71a1.
00000400:00000010:2.0:1787866217.065449:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000392966ae.
00000400:00000010:2.0:1787866217.065450:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a48aacda.
00000400:00000010:2.0:1787866217.065454:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005604167f.
00000400:00000010:2.0:1787866217.065456:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000005365e82e.
00000400:00000010:2.0:1787866217.065457:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000007e6773b2.
00000400:00000010:2.0:1787866217.065459:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000011714931.
00000400:00000010:2.0:1787866217.065467:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000049f79d76.
00000400:00000010:2.0:1787866217.065470:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000087818236.
00000400:00000010:2.0:1787866217.065471:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000008c8a7b83.
00000400:00000010:2.0:1787866217.065473:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000098227a4a.
00000400:00000010:2.0:1787866217.065476:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000028b9089e.
00000400:00000010:2.0:1787866217.065478:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000938cb033.
00000400:00000010:2.0:1787866217.065480:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000012a70cfa.
00000400:00000010:2.0:1787866217.065481:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000905a879b.
00000400:00000010:2.0:1787866217.065493:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b6fb7914.
00000400:00000010:2.0:1787866217.065496:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e9590d32.
00000400:00000010:2.0:1787866217.065498:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000b8459f62.
00000400:00000010:2.0:1787866217.065500:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009f2026b8.
00000400:00000010:2.0:1787866217.065503:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005e83f381.
00000400:00000010:2.0:1787866217.065506:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e4649c08.
00000400:00000010:2.0:1787866217.065507:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002d9eae7b.
00000400:00000010:2.0:1787866217.065509:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000006b193b9.
00000400:00000010:2.0:1787866217.065513:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002aacc980.
00000400:00000010:2.0:1787866217.065514:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000b2a3b6a8.
00000400:00000010:2.0:1787866217.065516:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000133b8b17.
00000400:00000010:2.0:1787866217.065517:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003ae1401c.
00000400:00000010:2.0:1787866217.065521:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000000861471c.
00000400:00000010:2.0:1787866217.065523:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000003ad4337c.
00000400:00000010:2.0:1787866217.065525:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000008f8e42b6.
00000400:00000010:2.0:1787866217.065526:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b8140383.
00000400:00000010:2.0:1787866217.065536:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005083cb8a.
00000400:00000010:2.0:1787866217.065539:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000012d03674.
00000400:00000010:2.0:1787866217.065541:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000bb5702e1.
00000400:00000010:2.0:1787866217.065543:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000069defbed.
00000400:00000010:2.0:1787866217.065547:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008198bbcd.
00000400:00000010:2.0:1787866217.065549:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000009ffd5ad9.
00000400:00000010:2.0:1787866217.065551:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000051c6a6ba.
00000400:00000010:2.0:1787866217.065552:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007e8c75e4.
00000400:00000010:2.0:1787866217.065560:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f29e49df.
00000400:00000010:2.0:1787866217.065562:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000008787625a.
00000400:00000010:2.0:1787866217.065564:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a7e11a58.
00000400:00000010:2.0:1787866217.065566:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000080bac632.
00000400:00000010:2.0:1787866217.065570:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000063414f75.
00000400:00000010:2.0:1787866217.065572:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000016ffbbec.
00000400:00000010:2.0:1787866217.065574:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000808ee949.
00000400:00000010:2.0:1787866217.065576:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b83e4947.
00000400:00000010:2.0:1787866217.065585:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006d892fb3.
00000400:00000010:2.0:1787866217.065589:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000070635d03.
00000400:00000010:2.0:1787866217.065591:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ca4b56be.
00000400:00000010:2.0:1787866217.065593:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000eb5843d2.
00000400:00000010:2.0:1787866217.065596:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000072a8f16f.
00000400:00000010:2.0:1787866217.065599:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a3257d5d.
00000400:00000010:2.0:1787866217.065601:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a629319d.
00000400:00000010:2.0:1787866217.065602:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a063f889.
00000400:00000010:2.0:1787866217.065606:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000076d6cd1c.
00000400:00000010:2.0:1787866217.065608:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000072de1828.
00000400:00000010:2.0:1787866217.065610:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000004d1fa072.
00000400:00000010:2.0:1787866217.065611:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f0782cce.
00000400:00000010:2.0:1787866217.065614:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000015c7a150.
00000400:00000010:2.0:1787866217.065616:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000072029be1.
00000400:00000010:2.0:1787866217.065618:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000088c059cd.
00000400:00000010:2.0:1787866217.065619:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ea07f75a.
00000400:00000010:2.0:1787866217.065629:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a3e62e9d.
00000400:00000010:2.0:1787866217.065632:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a4918ca0.
00000400:00000010:2.0:1787866217.065635:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000003dbeef4f.
00000400:00000010:2.0:1787866217.065637:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d72f4d76.
00000400:00000010:2.0:1787866217.065640:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000799598b3.
00000400:00000010:2.0:1787866217.065642:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ab15aaf2.
00000400:00000010:2.0:1787866217.065644:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000028d8caf.
00000400:00000010:2.0:1787866217.065646:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b140c2da.
00000400:00000010:2.0:1787866217.065655:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e53a5eee.
00000400:00000010:2.0:1787866217.065658:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000fd0ac38c.
00000400:00000010:2.0:1787866217.065660:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000021f78ebd.
00000400:00000010:2.0:1787866217.065662:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000b07253f.
00000400:00000010:2.0:1787866217.065666:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000bce067fc.
00000400:00000010:2.0:1787866217.065668:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000aaa9c056.
00000400:00000010:2.0:1787866217.065669:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e04eb8a3.
00000400:00000010:2.0:1787866217.065670:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003d4a94d8.
00000400:00000010:2.0:1787866217.065680:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000052006a4e.
00000400:00000010:2.0:1787866217.065684:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000007ae10c44.
00000400:00000010:2.0:1787866217.065685:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000004b0df6.
00000400:00000010:2.0:1787866217.065687:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000aa19f1a3.
00000400:00000010:2.0:1787866217.065691:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006ce923da.
00000400:00000010:2.0:1787866217.065693:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000be2546b6.
00000400:00000010:2.0:1787866217.065694:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000bbc4fead.
00000400:00000010:2.0:1787866217.065696:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c6497b0c.
00000400:00000010:2.0:1787866217.065699:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000067e117b8.
00000400:00000010:2.0:1787866217.065701:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000048d6c5d0.
00000400:00000010:2.0:1787866217.065703:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000f232dcd5.
00000400:00000010:2.0:1787866217.065705:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009c022a70.
00000400:00000010:2.0:1787866217.065708:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003dca47e3.
00000400:00000010:2.0:1787866217.065710:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000036b3a50e.
00000400:00000010:2.0:1787866217.065712:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000369e4733.
00000400:00000010:2.0:1787866217.065713:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000df0a1b16.
00000400:00000010:2.0:1787866217.065721:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ff320cb3.
00000400:00000010:2.0:1787866217.065724:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000375896ff.
00000400:00000010:2.0:1787866217.065725:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000506c05b3.
00000400:00000010:2.0:1787866217.065727:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f9a8cb83.
00000400:00000010:2.0:1787866217.065730:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000010775746.
00000400:00000010:2.0:1787866217.065732:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000012c6c579.
00000400:00000010:2.0:1787866217.065734:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000c1a9aa8c.
00000400:00000010:2.0:1787866217.065736:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a17cc9d9.
00000400:00000010:2.0:1787866217.065742:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006fe7da78.
00000400:00000010:2.0:1787866217.065744:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000013205a57.
00000400:00000010:2.0:1787866217.065746:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000057841de5.
00000400:00000010:2.0:1787866217.065748:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000aa4f13b0.
00000400:00000010:2.0:1787866217.065751:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f68aaa16.
00000400:00000010:2.0:1787866217.065754:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000007bf2fb4c.
00000400:00000010:2.0:1787866217.065755:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000000899e8cc.
00000400:00000010:2.0:1787866217.065757:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000055425607.
00000400:00000010:2.0:1787866217.065764:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000066cc6be4.
00000400:00000010:2.0:1787866217.065766:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000094312786.
00000400:00000010:2.0:1787866217.065768:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ffd0d466.
00000400:00000010:2.0:1787866217.065769:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d7a60416.
00000400:00000010:2.0:1787866217.065773:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d94bb90a.
00000400:00000010:2.0:1787866217.065775:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000aa387aa5.
00000400:00000010:2.0:1787866217.065776:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000008035aa29.
00000400:00000010:2.0:1787866217.065778:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007c81eb09.
00000400:00000010:2.0:1787866217.065781:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002f504b20.
00000400:00000010:2.0:1787866217.065783:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000088e7fc31.
00000400:00000010:2.0:1787866217.065784:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000bad506c6.
00000400:00000010:2.0:1787866217.065786:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d28068b0.
00000400:00000010:2.0:1787866217.065789:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f3b8bfe7.
00000400:00000010:2.0:1787866217.065791:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000c2ec0b30.
00000400:00000010:2.0:1787866217.065793:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002b653e15.
00000400:00000010:2.0:1787866217.065794:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e80ae833.
00000400:00000010:2.0:1787866217.065802:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007666a5df.
00000400:00000010:2.0:1787866217.065804:0:69373:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ae718185.
00000400:00000010:2.0:1787866217.065806:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000009467422c.
00000400:00000010:2.0:1787866217.065808:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000021031acc.
00000400:00000010:2.0:1787866217.065811:0:69373:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004da4d315.
00000400:00000010:2.0:1787866217.065814:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000026f6c9c0.
00000400:00000010:2.0:1787866217.065815:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005b139ed2.
00000400:00000010:2.0:1787866217.065819:0:69373:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 00000000481f47bd.
00000400:00000010:2.0:1787866217.065822:0:69373:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 552 at 00000000b026a959.
00000400:00000010:2.0:1787866217.065827:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005586a200.
00000400:00000010:2.0:1787866217.065828:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000027b2bdf8.
00000400:00000010:2.0:1787866217.066027:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ef2cd2ea.
00000400:00000010:2.0:1787866217.066029:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000054555ad8.
00000400:00000010:2.0:1787866217.066042:0:69373:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 41 at 0000000095220e4a.
00000400:00000010:2.0:1787866217.066046:0:69373:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000073d4377f.
00000400:00000010:2.0:1787866217.066058:0:69373:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000c5cf3b81.
00000400:00000200:2.0:1787866217.066067:0:69373:0:(lib-ptl.c:918:LNetSetLazyPortal()) Setting portal 0 lazy
00000400:00000010:2.0:1787866217.066070:0:69373:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000004ece727.
00000400:00000010:2.0:1787866217.066072:0:69373:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000910917a8.
00000400:00000010:2.0:1787866217.066074:0:69373:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001d03c3a2.
00000400:00000200:2.0:1787866217.066076:0:69373:0:(api-ni.c:2383:lnet_push_target_post()) posted push target 0000000004ece727
00000400:00000200:2.0:1787866217.066078:0:69373:0:(api-ni.c:2340:lnet_push_target_resize()) nbytes 272 success
00000400:00000010:2.0:1787866217.066080:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000023cd7e93.
00000400:00000010:2.0:1787866217.066081:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f95cda80.
00000400:00000010:2.0:1787866217.066084:0:69373:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000008876f365.
00000400:00000010:2.0:1787866217.066085:0:69373:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d9b6c465.
00000400:00000200:3.0:1787866217.066979:0:69373:0:(peer.c:4457:lnet_peer_discovery_start()) discovery start: 0
00000400:00000200:2.0:1787866217.067328:0:69375:0:(peer.c:4283:lnet_peer_discovery()) started
00000400:00000010:1.0:1787866217.085566:0:69376:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 0000000067e5948c.
00000400:00000010:1.0:1787866217.085588:0:69376:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 0000000067e5948c.
00000400:00000010:1.0:1787866217.086782:0:69376:0:(api-ni.c:7409:lnet_route_show_start()) alloc '(rlist)': 16 at 00000000e97aa57b.
00000400:00000010:1.0:1787866217.086806:0:69376:0:(api-ni.c:7309:lnet_route_show_done()) kfreed 'rlist': 16 at 00000000e97aa57b.
00000400:00000010:0.0:1787866217.089449:0:69376:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 236 at 00000000f46aa85a.
00000400:00000010:0.0:1787866217.089462:0:69376:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 236 at 00000000f46aa85a.
00000400:00000010:0.0:1787866217.089691:0:69376:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 00000000c2b45796.
00000400:00000010:0.0:1787866217.089707:0:69376:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 00000000c2b45796.
00000400:00000010:0.0:1787866217.089858:0:69376:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 12 at 00000000c2b45796.
00000400:00000010:0.0:1787866217.089864:0:69376:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 12 at 00000000c2b45796.
00000400:00000010:0.0:1787866217.090364:0:69376:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 32 at 00000000b67272b9.
00000001:00000200:0.0:1787866217.090369:0:69376:0:(udsp.c:863:lnet_udsp_get_policy()) Get UDSP at idx = 0
00000001:00000200:0.0:1787866217.090371:0:69376:0:(udsp.c:878:lnet_udsp_get_policy()) Found UDSP (00000000fad3d7f9)
00000400:00000010:0.0:1787866217.090373:0:69376:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 32 at 00000000b67272b9.
00000001:02000400:0.0:1787866217.097162:0:69377:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp --if ens2
00000400:00000010:2.0:1787866217.123531:0:69378:0:(api-ni.c:6877:lnet_net_cmd()) alloc '(conf)': 808 at 000000008771fcc0.
00000400:00000010:2.0:1787866217.123549:0:69378:0:(api-ni.c:6555:lnet_genl_parse_local_ni()) alloc '(tun)': 172 at 00000000721b7715.
00000400:00000010:2.0:1787866217.123558:0:69378:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 000000004ebeb1be.
00000400:00000010:2.0:1787866217.123562:0:69378:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 552 at 00000000dd651a5c.
00000400:00000010:2.0:1787866217.123566:0:69378:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000006435fc69.
00000400:00000010:2.0:1787866217.123570:0:69378:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b90a4287.
00000400:00000010:2.0:1787866217.123572:0:69378:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000cd87e7a3.
00000400:00000010:2.0:1787866217.123573:0:69378:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000075b49a8d.
00000400:00000010:2.0:1787866217.123578:0:69378:0:(config.c:397:lnet_ni_add_interface()) alloc '(ni->ni_interface)': 5 at 00000000ee9673b3.
00000400:00000010:2.0:1787866217.123586:0:69378:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000006ef8a308.
00000400:00000010:2.0:1787866217.123589:0:69378:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000891e7618.
00000800:00000200:3.0:1787866217.125722:0:69378:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=lo ifindex 1 state 0 ns 00000000198b8817
00000800:00000200:3.0:1787866217.125730:0:69378:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=lo ifindex 1 state 0 ns 00000000198b8817
00000800:00000200:3.0:1787866217.125733:0:69378:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=ens2 ifindex 2 state 6 ns 00000000198b8817
00000800:00000200:3.0:1787866217.125735:0:69378:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=ens2 ifindex 2 state 6 ns 00000000198b8817
00000800:00000010:3.0:1787866217.125743:0:69378:0:(socklnd.c:2621:ksocknal_startup()) alloc '(net)': 208 at 000000001b022c72.
00000800:00000200:3.0:1787866217.125757:0:69378:0:(socklnd_modparams.c:295:ksocklnd_lookup_conns_per_peer()) intf ens2 speed -1
00000400:00000200:3.0:1787866217.126775:0:69378:0:(config.c:1657:lnet_get_link_status_locked()) device idx 1 get_link 2
00000400:00000200:3.0:1787866217.126862:0:69378:0:(acceptor.c:209:lnet_acceptor_add_socket()) Added socket on ens2:988 (IP: 192.168.203.58)
00000400:02000000:3.0:1787866217.126877:0:69378:0:(api-ni.c:2823:lnet_startup_lndni()) Added LNI 192.168.203.58@tcp [8/256/0/180]
00000400:00000010:3.0:1787866217.131452:0:69378:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 57 at 00000000bd940ac1.
00000400:00000010:3.0:1787866217.131463:0:69378:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b3af47a7.
00000400:00000010:3.0:1787866217.131486:0:69378:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002dd861c7.
00000400:00000010:3.0:1787866217.132314:0:69378:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000073d4377f.
00000400:00000200:3.0:1787866217.132321:0:69378:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c5cf3b81
00000400:00000010:3.0:1787866217.132323:0:69378:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000c5cf3b81.
00000400:00000010:3.0:1787866217.132327:0:69378:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 41 at 0000000095220e4a.
00000400:00000010:3.0:1787866217.132365:0:69378:0:(api-ni.c:6852:lnet_genl_parse_local_ni()) kfreed 'tun': 172 at 00000000721b7715.
00000400:00000010:3.0:1787866217.132371:0:69378:0:(api-ni.c:6994:lnet_net_cmd()) kfreed 'conf': 808 at 000000008771fcc0.
00000400:00000010:1.0:1787866217.210216:0:69392:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000005d66f56a.
00000400:00000010:1.0:1787866217.210288:0:69392:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000005d66f56a.
00000001:02000400:1.0:1787866217.951131:0:69498:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp1 --if ens2
00000400:00000010:2.0:1787866217.971922:0:69499:0:(api-ni.c:6877:lnet_net_cmd()) alloc '(conf)': 808 at 0000000030e6b2cf.
00000400:00000010:2.0:1787866217.971941:0:69499:0:(api-ni.c:6555:lnet_genl_parse_local_ni()) alloc '(tun)': 172 at 00000000ff36d1a3.
00000400:00000010:2.0:1787866217.971951:0:69499:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 0000000060f85c02.
00000400:00000010:2.0:1787866217.971955:0:69499:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 552 at 000000008aa2db0b.
00000400:00000010:2.0:1787866217.971959:0:69499:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e143976c.
00000400:00000010:2.0:1787866217.971969:0:69499:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000000c2cb86.
00000400:00000010:2.0:1787866217.971971:0:69499:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000442aa5e4.
00000400:00000010:2.0:1787866217.971972:0:69499:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000066728e94.
00000400:00000010:2.0:1787866217.971976:0:69499:0:(config.c:397:lnet_ni_add_interface()) alloc '(ni->ni_interface)': 5 at 00000000d0789159.
00000800:00000010:2.0:1787866217.971983:0:69499:0:(socklnd.c:2621:ksocknal_startup()) alloc '(net)': 208 at 00000000e03769d4.
00000800:00000200:2.0:1787866217.971993:0:69499:0:(socklnd_modparams.c:295:ksocklnd_lookup_conns_per_peer()) intf ens2 speed -1
00000400:00000200:2.0:1787866217.972000:0:69499:0:(config.c:1657:lnet_get_link_status_locked()) device idx 1 get_link 2
00000400:00000200:2.0:1787866217.972045:0:69499:0:(acceptor.c:209:lnet_acceptor_add_socket()) Added socket on ens2:988 (IP: 192.168.203.58)
00000400:02000000:2.0:1787866217.972120:0:69499:0:(api-ni.c:2823:lnet_startup_lndni()) Added LNI 192.168.203.58@tcp1 [8/256/0/180]
00000400:00000010:2.0:1787866217.974626:0:69499:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 73 at 000000009dea2467.
00000400:00000010:2.0:1787866217.974640:0:69499:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000000061a5be.
00000400:00000010:2.0:1787866217.974647:0:69499:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000000c004c47.
00000400:00000010:2.0:1787866217.974662:0:69499:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b3af47a7.
00000400:00000200:2.0:1787866217.974668:0:69499:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002dd861c7
00000400:00000010:2.0:1787866217.974671:0:69499:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000002dd861c7.
00000400:00000010:2.0:1787866217.974677:0:69499:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 57 at 00000000bd940ac1.
00000400:00000010:2.0:1787866217.974699:0:69499:0:(api-ni.c:6852:lnet_genl_parse_local_ni()) kfreed 'tun': 172 at 00000000ff36d1a3.
00000400:00000010:2.0:1787866217.974705:0:69499:0:(api-ni.c:6994:lnet_net_cmd()) kfreed 'conf': 808 at 0000000030e6b2cf.
00000400:00000010:3.0:1787866218.041278:0:69504:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000005f02433f.
00000400:00000010:3.0:1787866218.041343:0:69504:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000005f02433f.
00000400:00000010:3.0:1787866218.836367:0:69623:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000005f02433f.
00000400:00000010:3.0:1787866218.836465:0:69623:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000005f02433f.
00000001:02000400:1.0:1787866218.861749:0:69625:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl discover 192.168.203.58@tcp
00000400:00000010:3.0:1787866218.885427:0:69626:0:(peer.c:230:lnet_peer_alloc()) alloc '(lp)': 360 at 00000000bd1acbb4.
00000400:00000200:3.0:1787866218.885450:0:69626:0:(peer.c:266:lnet_peer_alloc()) 00000000bd1acbb4 nid 192.168.203.58@tcp
00000400:00000010:3.0:1787866218.885454:0:69626:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 00000000548ae7a0.
00000400:00000200:3.0:1787866218.885457:0:69626:0:(peer.c:203:lnet_peer_net_alloc()) 00000000548ae7a0 net tcp
00000400:00000010:3.0:1787866218.885460:0:69626:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 0000000060c33e75.
00000400:00000200:3.0:1787866218.885464:0:69626:0:(peer.c:184:lnet_peer_ni_alloc()) 0000000060c33e75 nid 192.168.203.58@tcp
00000400:00000200:3.0:1787866218.885471:0:69626:0:(peer.c:1804:lnet_peer_attach_peer_ni()) peer 192.168.203.58@tcp NID 192.168.203.58@tcp flags 0x0
00000400:00000200:3.0:1787866218.885482:0:69626:0:(peer.c:2183:lnet_peer_ni_traffic_add()) peer 192.168.203.58@tcp: 0
00000400:00000200:3.0:1787866218.885583:0:69626:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.203.58@tcp: 0
00000400:00000200:3.0:1787866218.885586:0:69626:0:(peer.c:2935:lnet_discover_peer_locked()) Discovery attempt # 1
00000400:00000200:2.0:1787866218.889227:0:69375:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0
00000400:00000200:2.0:1787866218.889255:0:69375:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000bd1acbb4) state 0x40
00000400:00000010:2.0:1787866218.889271:0:69375:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000009ba206dd.
00000400:00000010:2.0:1787866218.889279:0:69375:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004b9e5ded.
00000400:00000010:2.0:1787866218.889321:0:69375:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000089805c47.
00000400:00000200:2.0:1787866218.889326:0:69375:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.203.58@tcp
00000400:00000200:2.0:1787866218.889343:0:69375:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR:  192.168.203.58@tcp local destination
00000400:00000200:2.0:1787866218.889351:0:69375:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.58@tcp
00000400:00000200:2.0:1787866218.889355:0:69375:0:(lib-move.c:1874:lnet_set_non_mr_pref_nid()) Setting preferred local NID 192.168.203.58@tcp on NMR peer 192.168.203.58@tcp
00000400:00000200:2.0:1787866218.889358:0:69375:0:(peer.c:1171:lnet_peer_ni_set_non_mr_pref_nid()) peer 192.168.203.58@tcp nid 192.168.203.58@tcp: 0
00000400:00000200:2.0:1787866218.889362:0:69375:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [18:18:256:4294967295] 192.168.203.58@tcp LPNI seq info [1:1:8:4294967295]
00000400:00000200:2.0:1787866218.889370:0:69375:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:2.0:1787866218.889377:0:69375:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:<?>) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : GET try# 0
00000800:00000200:2.0:1787866218.889382:0:69375:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.203.58@tcp
00000800:00000010:2.0:1787866218.889396:0:69375:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000e79cb9e1.
00000800:00000010:2.0:1787866218.889408:0:69375:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 00000000ff36d1a3.
00000800:00000010:2.0:1787866218.889412:0:69375:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 00000000e80b4a01.
00000800:00000200:2.0:1787866218.889418:0:69375:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ff36d1a3] -> 12345-192.168.203.58@tcp (2)
00000800:00000200:2.1:1787866218.889422:0:69375:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ff36d1a3] -> 12345-192.168.203.58@tcp (2)
00000400:00000200:2.0:1787866218.889478:0:69375:0:(peer.c:4022:lnet_peer_send_ping()) peer 192.168.203.58@tcp
00000400:00000200:2.0:1787866218.889481:0:69375:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000bd1acbb4) state 0x240 rc 0
00000800:00000010:0.0:1787866218.897233:0:69379:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 000000008fe39939.
00000800:00000010:0.0:1787866218.897260:0:69379:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 000000009ad80dbc.
00000800:00000010:0.0:1787866218.897269:0:69379:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 000000008cf6d149.
00000800:00000010:0.0:1787866218.897324:0:69379:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 000000008cf6d149.
00000400:00000200:2.0:1787866218.897509:0:69387:0:(acceptor.c:578:lnet_acceptor()) Accepted connection on ens2:988
00000400:00000200:2.0:1787866218.897534:0:69387:0:(acceptor.c:502:lnet_accept()) Accept 192.168.203.58@tcp from 255.3.0.2
00000800:00000010:2.0:1787866218.897538:0:69387:0:(socklnd.c:737:ksocknal_accept()) alloc '(cr)': 32 at 0000000097374f24.
00000800:00000010:3.0:1787866218.897624:0:69380:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 0000000085e71ae7.
00000800:00000010:3.0:1787866218.897630:0:69380:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 00000000ebcde6e1.
00000800:00000010:3.0:1787866218.897649:0:69380:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 00000000e9935768.
00000800:00000200:3.1:1787866218.897655:0:69380:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ff36d1a3] -> 12345-192.168.203.58@tcp (2)
00000800:00000200:3.1:1787866218.897661:0:69380:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.203.58@tcp 00000000e9935768 deleted
00000800:00000010:3.1:1787866218.897664:0:69380:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 00000000e9935768.
00000800:00000200:3.1:1787866218.897670:0:69380:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1
00000800:00000200:3.1:1787866218.897677:0:69380:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000200:3.1:1787866218.897683:0:69380:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000e79cb9e1 type 2, nob 96 niov 1 nkiov 0
00000800:00000200:3.0:1787866218.897688:0:69380:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.203.58@tcp p 3.x 192.168.203.58 -> 192.168.203.58:1023 incarnation:1787866217125745535 sched[0]
00000800:00000010:3.0:1787866218.897694:0:69380:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 000000001d65a08f.
00000800:00000010:3.0:1787866218.897775:0:69380:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 000000001d65a08f.
00000800:00000010:3.0:1787866218.897780:0:69380:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 00000000ebcde6e1.
00000800:00000010:3.0:1787866218.897819:0:69380:0:(socklnd_cb.c:2179:ksocknal_connd()) kfreed 'cr': 32 at 0000000097374f24.
00000800:00000200:1.0:1787866218.897943:0:69385:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1787866218.897948:0:69385:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000e79cb9e1.
00000400:00000200:1.0:1787866218.897953:0:69385:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1787866218.897959:0:69385:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: GET: OK
00000400:00000200:1.0:1787866218.897966:0:69385:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 5
00000400:00000200:1.0:1787866218.897970:0:69385:0:(peer.c:3241:lnet_discovery_event_send()) Ping Send to 192.168.203.58@tcp: 0
00000800:00000200:0.1:1787866218.898228:0:69379:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1
00000800:00000200:0.0:1787866218.898234:0:69379:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.203.58@tcp p 3.x 192.168.203.58 -> 192.168.203.58:988 incarnation:1787866217125745535 sched[0]
00000800:00000010:0.0:1787866218.898238:0:69379:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 000000009ad80dbc.
00000400:00000200:0.0:1787866218.898339:0:69379:0:(lib-socket.c:438:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.203.58:988
00000800:00000010:0.0:1787866218.898564:0:69379:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 000000008d8cdc37.
00000800:00000010:0.0:1787866218.898568:0:69379:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 000000009ad80dbc.
00000800:00000010:0.0:1787866218.898572:0:69379:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 000000008cf6d149.
00000800:00000010:0.0:1787866218.898598:0:69379:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 000000008cf6d149.
00000400:00000200:2.0:1787866218.898703:0:69386:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : GET - for me
00000400:00000200:2.0:1787866218.898720:0:69386:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.58@tcp of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:2.0:1787866218.898728:0:69386:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.203.58@tcp of length 64/272 into md 0xd [1] + 0
00000400:00000200:2.0:1787866218.898752:0:69386:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.203.58@tcp to NMR:  192.168.203.58@tcp local destination
00000400:00000200:2.0:1787866218.898758:0:69386:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [19:19:256:4294967295] 192.168.203.58@tcp LPNI seq info [2:2:8:4294967295]
00000400:00000200:2.0:1787866218.898769:0:69386:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) 192.168.203.58@tcp : REPLY try# 0
00000800:00000200:2.0:1787866218.898775:0:69386:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.203.58@tcp
00000800:00000010:2.0:1787866218.898781:0:69386:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000004bd49c44.
00000800:00000200:2.0:1787866218.898786:0:69386:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ff36d1a3] -> 12345-192.168.203.58@tcp (4)
00000800:00000200:2.0:1787866218.898793:0:69386:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000200:2.0:1787866218.898796:0:69386:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000004bd49c44 type 3, nob 160 niov 1 nkiov 1
00000800:00000200:2.0:1787866218.898885:0:69386:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1787866218.898889:0:69386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000004bd49c44.
00000400:00000200:2.0:1787866218.898892:0:69386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1787866218.898897:0:69386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: REPLY: OK
00000400:00000200:2.0:1787866218.898915:0:69386:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : REPLY - for me
00000400:00000200:2.0:1787866218.898924:0:69386:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.203.58@tcp: Reply from 12345-192.168.203.58@tcp of length 64/64 into md 0x11
00000400:00000200:2.0:1787866218.898931:0:69386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1787866218.898934:0:69386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: REPLY: OK
00000400:00000200:2.0:1787866218.898938:0:69386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004b9e5ded
00000400:00000010:2.0:1787866218.898947:0:69386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000004b9e5ded.
00000400:00000200:2.0:1787866218.898951:0:69386:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 3
00000400:00000200:2.0:1787866218.898955:0:69386:0:(peer.c:3119:lnet_discovery_event_reply()) Peer 192.168.203.58@tcp has discovery enabled
00000400:00000200:2.0:1787866218.898959:0:69386:0:(peer.c:3145:lnet_discovery_event_reply()) peer 192.168.203.58@tcp(00000000bd1acbb4) is MR capable
00000400:00000200:2.0:1787866218.898963:0:69386:0:(peer.c:1196:lnet_peer_ni_clr_non_mr_pref_nid()) peer 192.168.203.58@tcp: 0
00000400:00000200:2.0:1787866218.898966:0:69386:0:(peer.c:3200:lnet_discovery_event_reply()) peer 192.168.203.58@tcp data present 9. state = 0x241
00000400:00000200:2.0:1787866218.898971:0:69386:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.203.58@tcp: -114
00000400:00000200:2.0:1787866218.898993:0:69375:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0
00000400:00000200:2.0:1787866218.898999:0:69375:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000bd1acbb4) state 0xc1
00000400:00000010:2.0:1787866218.899006:0:69375:0:(peer.c:3474:lnet_peer_merge_data()) alloc '(curnis)': 60 at 000000005ee7d883.
00000400:00000010:2.0:1787866218.899010:0:69375:0:(peer.c:3475:lnet_peer_merge_data()) alloc '(addnis)': 72 at 0000000019d8f1d6.
00000400:00000010:2.0:1787866218.899012:0:69375:0:(peer.c:3476:lnet_peer_merge_data()) alloc '(delnis)': 60 at 0000000060bf75e9.
00000400:00000010:2.0:1787866218.899021:0:69375:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 000000008ee7702b.
00000400:00000200:2.0:1787866218.899025:0:69375:0:(peer.c:184:lnet_peer_ni_alloc()) 000000008ee7702b nid 192.168.203.58@tcp1
00000400:00000010:2.0:1787866218.899028:0:69375:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 0000000010a6e38f.
00000400:00000200:2.0:1787866218.899031:0:69375:0:(peer.c:203:lnet_peer_net_alloc()) 0000000010a6e38f net tcp1
00000400:00000200:2.0:1787866218.899038:0:69375:0:(peer.c:1804:lnet_peer_attach_peer_ni()) peer 192.168.203.58@tcp NID 192.168.203.58@tcp1 flags 0x11
00000400:00000010:2.0:1787866218.899042:0:69375:0:(peer.c:3623:lnet_peer_merge_data()) kfreed 'curnis': 60 at 000000005ee7d883.
00000400:00000010:2.0:1787866218.899046:0:69375:0:(peer.c:3624:lnet_peer_merge_data()) kfreed 'addnis': 72 at 0000000019d8f1d6.
00000400:00000010:2.0:1787866218.899048:0:69375:0:(peer.c:3625:lnet_peer_merge_data()) kfreed 'delnis': 60 at 0000000060bf75e9.
00000400:00000010:2.0:1787866218.899051:0:69375:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000009ba206dd.
00000400:00000200:2.0:1787866218.899054:0:69375:0:(peer.c:3628:lnet_peer_merge_data()) peer 192.168.203.58@tcp (00000000bd1acbb4): 0
00000400:00000200:2.0:1787866218.899057:0:69375:0:(peer.c:3940:lnet_peer_data_present()) peer 192.168.203.58@tcp(00000000bd1acbb4): 0. state = 0x141
00000400:00000200:2.0:1787866218.899061:0:69375:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000bd1acbb4) state 0x141 rc 1
00000400:00000200:2.0:1787866218.899064:0:69375:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000bd1acbb4) state 0x141
00000400:00000010:2.0:1787866218.899070:0:69375:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004b9e5ded.
00000400:00000010:2.0:1787866218.899078:0:69375:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000d47bc20e.
00000400:00000200:2.0:1787866218.899082:0:69375:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.58@tcp
00000400:00000200:2.0:1787866218.899089:0:69375:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.203.58@tcp to MR:  192.168.203.58@tcp local destination
00000400:00000200:2.0:1787866218.899093:0:69375:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [20:20:256:4294967295] 192.168.203.58@tcp LPNI seq info [3:3:8:4294967295]
00000400:00000200:2.0:1787866218.899097:0:69375:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:2.0:1787866218.899103:0:69375:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787866218.899296:0:69375:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.203.58@tcp
00000800:00000010:2.0:1787866218.899300:0:69375:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000004bd49c44.
00000800:00000200:2.0:1787866218.899305:0:69375:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ff36d1a3] -> 12345-192.168.203.58@tcp (4)
00000800:00000200:2.0:1787866218.899309:0:69375:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000200:2.0:1787866218.899311:0:69375:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000004bd49c44 type 1, nob 160 niov 1 nkiov 1
00000400:00000200:2.0:1787866218.899362:0:69375:0:(peer.c:4164:lnet_peer_send_push()) peer 192.168.203.58@tcp
00000400:00000200:2.0:1787866218.899365:0:69375:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000bd1acbb4) state 0x541 rc 0
00000400:00000200:3.0:1787866218.899517:0:69387:0:(acceptor.c:578:lnet_acceptor()) Accepted connection on ens2:988
00000400:00000200:3.0:1787866218.899529:0:69387:0:(acceptor.c:502:lnet_accept()) Accept 192.168.203.58@tcp from 254.3.0.2
00000800:00000010:3.0:1787866218.899532:0:69387:0:(socklnd.c:737:ksocknal_accept()) alloc '(cr)': 32 at 0000000033d45d63.
00000800:00000010:2.0:1787866218.899619:0:69381:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 00000000eec38694.
00000800:00000010:2.0:1787866218.899623:0:69381:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 00000000721b7715.
00000800:00000010:2.0:1787866218.899637:0:69381:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 00000000ddaae30f.
00000800:00000200:2.1:1787866218.899641:0:69381:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ff36d1a3] -> 12345-192.168.203.58@tcp (4)
00000800:00000200:2.1:1787866218.899646:0:69381:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.203.58@tcp 00000000ddaae30f deleted
00000800:00000010:2.1:1787866218.899648:0:69381:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 00000000ddaae30f.
00000800:00000200:2.1:1787866218.899651:0:69381:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 3, ksnr_connected a ksnr_max_conns 1
00000800:00000200:2.0:1787866218.899655:0:69381:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.203.58@tcp p 3.x 192.168.203.58 -> 192.168.203.58:1022 incarnation:1787866217125745535 sched[0]
00000800:00000010:2.0:1787866218.899661:0:69381:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 0000000060bf75e9.
00000800:00000010:2.0:1787866218.899743:0:69381:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 0000000060bf75e9.
00000800:00000010:2.0:1787866218.899747:0:69381:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 00000000721b7715.
00000800:00000200:3.1:1787866218.899794:0:69379:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 2, ksnr_connected e ksnr_max_conns 1
00000800:00000200:3.0:1787866218.899800:0:69379:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.203.58@tcp p 3.x 192.168.203.58 -> 192.168.203.58:988 incarnation:1787866217125745535 sched[0]
00000800:00000010:3.0:1787866218.899803:0:69379:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 000000009ad80dbc.
00000800:00000200:1.0:1787866218.905565:0:69386:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1787866218.905577:0:69386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000004bd49c44.
00000400:00000200:1.0:1787866218.905585:0:69386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1787866218.905596:0:69386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: PUT: OK
00000400:00000200:1.0:1787866218.905604:0:69386:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 5
00000400:00000200:1.0:1787866218.905608:0:69386:0:(peer.c:3241:lnet_discovery_event_send()) Push Send to 192.168.203.58@tcp: 0
00000400:00000200:1.0:1787866218.905637:0:69386:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : PUT - for me
00000400:00000200:1.0:1787866218.905683:0:69386:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.58@tcp of length 64 into portal 0 MB=0x8000000000000000
00000400:00000200:1.0:1787866218.905693:0:69386:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 0 from 12345-192.168.203.58@tcp of length 64/64 into md 0x5 [1] + 0
00000400:00000010:1.0:1787866218.905698:0:69386:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000910917a8.
00000400:00000200:1.0:1787866218.905703:0:69386:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000001d03c3a2
00000400:00000200:1.0:1787866218.905715:0:69386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1787866218.905718:0:69386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: PUT: OK
00000400:00000200:1.0:1787866218.905723:0:69386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001d03c3a2
00000400:00000010:1.0:1787866218.905725:0:69386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000001d03c3a2.
00000400:00000200:1.0:1787866218.905729:0:69386:0:(api-ni.c:2393:lnet_push_target_event_handler()) type 2 status 0 unlinked 1
00000400:00000200:1.0:1787866218.905940:0:69386:0:(peer.c:2778:lnet_peer_push_event()) peer 192.168.203.58@tcp(00000000bd1acbb4) is MR
00000400:00000010:1.0:1787866218.905947:0:69386:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 73 at 0000000016f17e78.
00000400:00000200:1.0:1787866218.905950:0:69386:0:(peer.c:2852:lnet_peer_push_event()) Received Push 192.168.203.58@tcp 9
00000400:00000200:1.0:1787866218.905955:0:69386:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.203.58@tcp: -114
00000400:00000200:1.0:1787866218.905979:0:69386:0:(lib-md.c:96:lnet_cpt_of_md()) Couldn't resolve first page of md 0000000000000000 with offset 0
00000400:00000200:1.0:1787866218.905987:0:69386:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.203.58@tcp to MR:  192.168.203.58@tcp local destination
00000400:00000200:1.0:1787866218.905994:0:69386:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [21:21:256:4294967295] 192.168.203.58@tcp LPNI seq info [4:4:8:4294967295]
00000400:00000200:1.0:1787866218.906006:0:69386:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) 192.168.203.58@tcp : ACK try# 0
00000800:00000200:1.0:1787866218.906012:0:69386:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 1 frags to 12345-192.168.203.58@tcp
00000800:00000010:1.0:1787866218.906018:0:69386:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002319231c.
00000800:00000200:1.0:1787866218.906025:0:69386:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ff36d1a3] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:1.0:1787866218.906033:0:69386:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000200:1.0:1787866218.906039:0:69386:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002319231c type 0, nob 96 niov 1 nkiov 0
00000800:00000200:1.0:1787866218.906087:0:69386:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1787866218.906089:0:69386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000002319231c.
00000400:00000200:1.0:1787866218.906092:0:69386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1787866218.906096:0:69386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: ACK: OK
00000400:00000200:1.0:1787866218.906109:0:69386:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : ACK - for me
00000400:00000200:1.0:1787866218.906129:0:69386:0:(lib-move.c:4539:lnet_parse_ack()) 192.168.203.58@tcp: ACK from 12345-192.168.203.58@tcp into md 0x15
00000400:00000200:1.0:1787866218.906133:0:69386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1787866218.906136:0:69386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: ACK: OK
00000400:00000200:1.0:1787866218.906139:0:69386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004b9e5ded
00000400:00000010:1.0:1787866218.906141:0:69386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000004b9e5ded.
00000400:00000200:1.0:1787866218.906144:0:69386:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 4
00000400:00000200:1.0:1787866218.906146:0:69386:0:(peer.c:3007:lnet_discovery_event_ack()) peer 192.168.203.58@tcp ev->status 0
00000400:00000200:1.0:1787866218.906149:0:69386:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.203.58@tcp: -114
00000400:00000200:3.0:1787866218.906420:0:69375:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0
00000400:00000010:3.0:1787866218.906429:0:69375:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000094649d34.
00000400:00000010:3.0:1787866218.906436:0:69375:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e4d53631.
00000400:00000200:3.0:1787866218.906443:0:69375:0:(api-ni.c:2383:lnet_push_target_post()) posted push target 0000000004ece727
00000400:00000200:3.0:1787866218.906449:0:69375:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000bd1acbb4) state 0xc1
00000400:00000010:3.0:1787866218.906457:0:69375:0:(peer.c:3474:lnet_peer_merge_data()) alloc '(curnis)': 60 at 000000001d65a08f.
00000400:00000010:3.0:1787866218.906461:0:69375:0:(peer.c:3475:lnet_peer_merge_data()) alloc '(addnis)': 72 at 00000000bc24b5b5.
00000400:00000010:3.0:1787866218.906463:0:69375:0:(peer.c:3476:lnet_peer_merge_data()) alloc '(delnis)': 60 at 0000000058ad283f.
00000400:00000010:3.0:1787866218.906470:0:69375:0:(peer.c:3623:lnet_peer_merge_data()) kfreed 'curnis': 60 at 000000001d65a08f.
00000400:00000010:3.0:1787866218.906473:0:69375:0:(peer.c:3624:lnet_peer_merge_data()) kfreed 'addnis': 72 at 00000000bc24b5b5.
00000400:00000010:3.0:1787866218.906476:0:69375:0:(peer.c:3625:lnet_peer_merge_data()) kfreed 'delnis': 60 at 0000000058ad283f.
00000400:00000010:3.0:1787866218.906479:0:69375:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 73 at 0000000016f17e78.
00000400:00000200:3.0:1787866218.906482:0:69375:0:(peer.c:3628:lnet_peer_merge_data()) peer 192.168.203.58@tcp (00000000bd1acbb4): 0
00000400:00000200:3.0:1787866218.906485:0:69375:0:(peer.c:3940:lnet_peer_data_present()) peer 192.168.203.58@tcp(00000000bd1acbb4): 0. state = 0x141
00000400:00000200:3.0:1787866218.906489:0:69375:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000bd1acbb4) state 0x141 rc 1
00000400:00000200:3.0:1787866218.906492:0:69375:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000bd1acbb4) state 0x141
00000400:00000200:3.0:1787866218.906502:0:69375:0:(peer.c:4082:lnet_peer_discovered()) peer 192.168.203.58@tcp
00000400:00000200:3.0:1787866218.906504:0:69375:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000bd1acbb4) state 0x111 rc 0
00000400:00000200:3.0:1787866218.906508:0:69375:0:(peer.c:2629:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 192.168.203.58@tcp
00000400:00000200:1.0:1787866218.906683:0:69626:0:(peer.c:2984:lnet_discover_peer_locked()) peer 192.168.203.58@tcp NID 192.168.203.58@tcp: 0. discovery complete
00000800:00000010:2.0:1787866218.908447:0:69381:0:(socklnd_cb.c:2179:ksocknal_connd()) kfreed 'cr': 32 at 0000000033d45d63.
00000400:00000010:3.0:1787866218.933709:0:69628:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 12 at 00000000df680292.
00000400:00000010:3.0:1787866218.933730:0:69628:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 12 at 00000000df680292.
00000001:02000400:0.0:1787866218.963197:0:69630:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 10
00000001:02000400:3.0:1787866219.012291:0:69632:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp -d *@tcp -m GET -r 1 -e remote_error
00000400:00000200:1.0:1787866219.037403:0:69633:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866219.037416:0:69633:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866219.037431:0:69633:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp
00000400:00000200:1.0:1787866219.037433:0:69633:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866219.037442:0:69633:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp
00000400:00000200:1.0:1787866219.037443:0:69633:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866219.037446:0:69633:0:(api-ni.c:3207:nla_strnid()) <?> -> <?>
00000400:00000200:1.0:1787866219.037447:0:69633:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866219.037449:0:69633:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866219.037451:0:69633:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866219.037452:0:69633:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866219.037454:0:69633:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866219.037456:0:69633:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866219.037458:0:69633:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000010:1.0:1787866219.037463:0:69633:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 000000008b8310b3.
00000400:00000200:1.0:1787866219.037473:0:69633:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp, dst 255.255.255.255@tcp, rate 1, interval 0
00000001:02000400:2.0:1787866219.063893:0:69634:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp1 -d *@tcp1 -m GET -r 1 -e remote_error
00000400:00000200:3.0:1787866219.097717:0:69635:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866219.097729:0:69635:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866219.099124:0:69635:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1
00000400:00000200:3.0:1787866219.099132:0:69635:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866219.099138:0:69635:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1
00000400:00000200:3.0:1787866219.099139:0:69635:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866219.099142:0:69635:0:(api-ni.c:3207:nla_strnid()) <?> -> <?>
00000400:00000200:3.0:1787866219.099143:0:69635:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866219.099144:0:69635:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866219.099146:0:69635:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866219.099147:0:69635:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866219.099149:0:69635:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866219.099150:0:69635:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866219.099152:0:69635:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000010:3.0:1787866219.099163:0:69635:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 000000002eacacef.
00000400:00000200:3.0:1787866219.099173:0:69635:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp1, dst 255.255.255.255@tcp1, rate 1, interval 0
00000001:02000400:2.0:1787866219.119566:0:69636:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net_drop add -s *@tcp -d *@tcp -r 1
00000400:00000200:0.0:1787866219.156168:0:69637:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866219.156182:0:69637:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866219.156197:0:69637:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp
00000400:00000200:0.0:1787866219.156200:0:69637:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866219.156203:0:69637:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp
00000400:00000200:0.0:1787866219.156204:0:69637:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866219.156207:0:69637:0:(api-ni.c:3207:nla_strnid()) <?> -> <?>
00000400:00000200:0.0:1787866219.156208:0:69637:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866219.156210:0:69637:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866219.156211:0:69637:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866219.156213:0:69637:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866219.156221:0:69637:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866219.156223:0:69637:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866219.156225:0:69637:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000010:0.0:1787866219.156230:0:69637:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 000000005df545ae.
00000400:00000200:0.0:1787866219.156250:0:69637:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp, dst 255.255.255.255@tcp, rate 1, interval 0
00000001:02000400:1.0:1787866219.189104:0:69638:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net_drop add -s *@tcp1 -d *@tcp1 -r 1
00000400:00000200:0.0:1787866219.225105:0:69639:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866219.225295:0:69639:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866219.225313:0:69639:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1
00000400:00000200:0.0:1787866219.225315:0:69639:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866219.225318:0:69639:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1
00000400:00000200:0.0:1787866219.225319:0:69639:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866219.225322:0:69639:0:(api-ni.c:3207:nla_strnid()) <?> -> <?>
00000400:00000200:0.0:1787866219.225324:0:69639:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866219.225325:0:69639:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866219.225326:0:69639:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866219.225327:0:69639:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866219.225329:0:69639:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866219.225330:0:69639:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866219.225333:0:69639:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000010:2.0:1787866219.225982:0:69639:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 00000000159023a2.
00000400:00000200:2.0:1787866219.225996:0:69639:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp1, dst 255.255.255.255@tcp1, rate 1, interval 0
00000001:02000400:1.0:1787866219.258794:0:69640:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --nid 192.168.203.58@tcp --health 0
00000400:00000200:2.0:1787866219.291339:0:69641:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.203.58@tcp added to recovery queue. ping count: 0 next ping: 1430 last alive: 1428 health: 0
00000400:00000200:1.0:1787866219.373380:0:69647:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:1.0:1787866219.373391:0:69647:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000004897fe73.
00000400:00000200:1.0:1787866219.373404:0:69647:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:1.0:1787866219.373431:0:69647:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 000000009c540dae.
00000400:00000200:1.0:1787866219.373440:0:69647:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:1.0:1787866219.373445:0:69647:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 000000009c540dae.
00000400:00000010:1.0:1787866219.373452:0:69647:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000004897fe73.
00000001:02000400:1.0:1787866219.417199:0:69649:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p
00000400:00000200:1.0:1787866219.446583:0:69650:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:1.0:1787866219.446593:0:69650:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 0000000001dd80f0.
00000400:00000200:1.0:1787866219.446608:0:69650:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:1.0:1787866219.446639:0:69650:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 000000004897fe73.
00000400:00000200:1.0:1787866219.446648:0:69650:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:1.0:1787866219.446653:0:69650:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 000000004897fe73.
00000400:00000010:1.0:1787866219.446663:0:69650:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 0000000001dd80f0.
00000400:00000200:2.0:1787866219.523924:0:69656:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:2.0:1787866219.523936:0:69656:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000f579002a.
00000400:00000200:2.0:1787866219.524347:0:69656:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:2.0:1787866219.524375:0:69656:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 000000003c4758a4.
00000400:00000200:2.0:1787866219.524383:0:69656:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:2.0:1787866219.524388:0:69656:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 000000003c4758a4.
00000400:00000010:2.0:1787866219.524398:0:69656:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000f579002a.
00000400:00000010:1.0:1787866220.179006:0:69374:0:(lib-move.c:3721:lnet_recover_peer_nis()) alloc '(ev_info)': 24 at 00000000a3684d5f.
00000400:00000010:1.0:1787866220.179027:0:69374:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000fb6cd7af.
00000400:00000010:1.0:1787866220.179063:0:69374:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000043e8f9e5.
00000400:00000010:1.0:1787866220.179076:0:69374:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000eefdcd83.
00000400:00000200:1.0:1787866220.179085:0:69374:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.203.58@tcp
00000400:00000200:1.0:1787866220.179106:0:69374:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR:  192.168.203.58@tcp local destination
00000400:00000200:1.0:1787866220.179113:0:69374:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.58@tcp
00000400:00000200:1.0:1787866220.179117:0:69374:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [22:22:256:4294967295] 192.168.203.58@tcp LPNI seq info [5:5:8:4294967295]
00000400:00000200:1.0:1787866220.179125:0:69374:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:1.0:1787866220.179132:0:69374:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:<?>) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : GET try# 0
00000800:00000200:1.0:1787866220.179138:0:69374:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.203.58@tcp
00000800:00000010:1.0:1787866220.179143:0:69374:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000002319231c.
00000800:00000200:1.0:1787866220.179150:0:69374:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ff36d1a3] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:1.0:1787866220.179159:0:69374:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000200:1.0:1787866220.179164:0:69374:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002319231c type 2, nob 96 niov 1 nkiov 0
00000400:00000200:1.0:1787866220.179185:0:69374:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.203.58@tcp added to recovery queue. ping count: 1 next ping: 1432 last alive: 1428 health: 0
00000400:00000010:1.0:1787866220.179191:0:69374:0:(lib-move.c:3814:lnet_print_recovery_list()) alloc '(buf)': 65 at 00000000211383d9.
00000400:02000040:1.0:1787866220.179194:0:69374:0:(lib-move.c:3830:lnet_print_recovery_list()) 1 peer NIs in recovery (showing 1): 192.168.203.58@tcp
00000400:00000200:2.0:1787866220.181389:0:69386:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 1
00000400:00000200:2.0:1787866220.181403:0:69386:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.203.58@tcp(192.168.203.58@tcp)->dst 192.168.203.58@tcp: GET simulate health error: REMOTE_ERROR
00000800:02000400:2.0:1787866220.181428:0:69386:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.203.58: rc = -22
00000400:00000010:1.0:1787866220.187180:0:69374:0:(lib-move.c:3832:lnet_print_recovery_list()) kfreed 'buf': 65 at 00000000211383d9.
00000800:00000200:2.0:1787866220.192331:0:69386:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [000000008fe39939] Error -22 on write to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000010:2.0:1787866220.192346:0:69386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000002319231c.
00000400:00000200:2.0:1787866220.192357:0:69386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787866220.192365:0:69386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: GET: REMOTE_ERROR
00000400:00000200:2.0:1787866220.192376:0:69386:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5
00000400:00000200:2.0:1787866220.192381:0:69386:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.203.58@tcp recovery message sent unsuccessfully:-5
00000400:00000200:2.0:1787866220.192388:0:69386:0:(lib-move.c:4104:lnet_handle_recovery_reply()) peer NI (192.168.203.58@tcp) recovery failed with -5
00000400:00000200:0.0:1787866220.660028:0:69664:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:0.0:1787866220.660039:0:69664:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000000f65dfed.
00000400:00000200:0.0:1787866220.660052:0:69664:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:0.0:1787866220.660075:0:69664:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000c898f900.
00000400:00000200:0.0:1787866220.660084:0:69664:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:0.0:1787866220.660089:0:69664:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000c898f900.
00000400:00000010:0.0:1787866220.660099:0:69664:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000000f65dfed.
00000400:00000200:1.0:1787866221.202980:0:69374:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000043e8f9e5
00000400:00000010:1.0:1787866221.202993:0:69374:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 0000000043e8f9e5.
00000400:00000200:1.0:1787866221.203002:0:69374:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0
00000400:00000200:1.0:1787866221.203010:0:69374:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.203.58@tcp recovery ping unlinked
00000400:00000010:1.0:1787866221.203017:0:69374:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000a3684d5f.
00000400:00000010:1.0:1787866221.203022:0:69374:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000fb6cd7af.
00000400:00000200:2.0:1787866221.775363:0:69672:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:2.0:1787866221.775376:0:69672:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000005ee7d883.
00000400:00000200:2.0:1787866221.775392:0:69672:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:2.0:1787866221.776090:0:69672:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000f579002a.
00000400:00000200:2.0:1787866221.776104:0:69672:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:2.0:1787866221.776110:0:69672:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000f579002a.
00000400:00000010:2.0:1787866221.776121:0:69672:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000005ee7d883.
00000400:00000010:1.0:1787866222.228396:0:69374:0:(lib-move.c:3721:lnet_recover_peer_nis()) alloc '(ev_info)': 24 at 00000000a3684d5f.
00000400:00000010:1.0:1787866222.228416:0:69374:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000fb6cd7af.
00000400:00000010:1.0:1787866222.228424:0:69374:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000043e8f9e5.
00000400:00000010:1.0:1787866222.228434:0:69374:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000008942af4d.
00000400:00000200:1.0:1787866222.228456:0:69374:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.203.58@tcp
00000400:00000200:1.0:1787866222.228483:0:69374:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR:  192.168.203.58@tcp local destination
00000400:00000200:1.0:1787866222.228491:0:69374:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.58@tcp
00000400:00000200:1.0:1787866222.228495:0:69374:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [23:23:256:4294967295] 192.168.203.58@tcp LPNI seq info [6:6:8:4294967295]
00000400:00000200:1.0:1787866222.228502:0:69374:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:1.0:1787866222.228509:0:69374:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:<?>) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : GET try# 0
00000800:00000200:1.0:1787866222.228534:0:69374:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.203.58@tcp
00000800:00000010:1.0:1787866222.228541:0:69374:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000002319231c.
00000800:00000200:1.0:1787866222.228549:0:69374:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ff36d1a3] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:1.0:1787866222.228558:0:69374:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000200:1.0:1787866222.228563:0:69374:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002319231c type 2, nob 96 niov 1 nkiov 0
00000400:00000200:1.0:1787866222.228590:0:69374:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.203.58@tcp added to recovery queue. ping count: 2 next ping: 1436 last alive: 1428 health: 0
00000400:00000200:3.0:1787866222.228746:0:69385:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 2
00000400:00000200:3.0:1787866222.228753:0:69385:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.203.58@tcp(192.168.203.58@tcp)->dst 192.168.203.58@tcp: GET simulate health error: REMOTE_ERROR
00000800:02000400:3.0:1787866222.228757:0:69385:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.203.58: rc = -22
00000800:00000200:3.0:1787866222.228761:0:69385:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [0000000085e71ae7] Error -22 on write to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000010:3.0:1787866222.228766:0:69385:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000002319231c.
00000400:00000200:3.0:1787866222.228775:0:69385:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787866222.228780:0:69385:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: GET: REMOTE_ERROR
00000400:00000200:3.0:1787866222.228787:0:69385:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5
00000400:00000200:3.0:1787866222.228790:0:69385:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.203.58@tcp recovery message sent unsuccessfully:-5
00000400:00000200:3.0:1787866222.228795:0:69385:0:(lib-move.c:4104:lnet_handle_recovery_reply()) peer NI (192.168.203.58@tcp) recovery failed with -5
00000400:00000200:1.0:1787866222.933360:0:69680:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:1.0:1787866222.933373:0:69680:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000e6ad20f0.
00000400:00000200:1.0:1787866222.933385:0:69680:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:1.0:1787866222.933410:0:69680:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000379f6d0e.
00000400:00000200:1.0:1787866222.933418:0:69680:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:1.0:1787866222.933422:0:69680:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000379f6d0e.
00000400:00000010:1.0:1787866222.933432:0:69680:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000e6ad20f0.
00000400:00000200:1.0:1787866223.251120:0:69374:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000043e8f9e5
00000400:00000010:1.0:1787866223.251133:0:69374:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 0000000043e8f9e5.
00000400:00000200:1.0:1787866223.251142:0:69374:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0
00000400:00000200:1.0:1787866223.251151:0:69374:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.203.58@tcp recovery ping unlinked
00000400:00000010:1.0:1787866223.251169:0:69374:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000a3684d5f.
00000400:00000010:1.0:1787866223.251175:0:69374:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000fb6cd7af.
00000400:00000200:3.0:1787866224.117870:0:69689:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:3.0:1787866224.117885:0:69689:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000006fd6188a.
00000400:00000200:3.0:1787866224.117899:0:69689:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:3.0:1787866224.117917:0:69689:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 0000000007fcdeb7.
00000400:00000200:3.0:1787866224.117924:0:69689:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:3.0:1787866224.117929:0:69689:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 0000000007fcdeb7.
00000400:00000010:3.0:1787866224.117937:0:69689:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000006fd6188a.
00000400:00000200:0.0:1787866225.234123:0:69697:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:0.0:1787866225.234135:0:69697:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000000f65dfed.
00000400:00000200:0.0:1787866225.234252:0:69697:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:0.0:1787866225.234275:0:69697:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000c898f900.
00000400:00000200:0.0:1787866225.234283:0:69697:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:0.0:1787866225.234288:0:69697:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000c898f900.
00000400:00000010:0.0:1787866225.234297:0:69697:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000000f65dfed.
00000400:00000010:0.0:1787866226.326908:0:69374:0:(lib-move.c:3721:lnet_recover_peer_nis()) alloc '(ev_info)': 24 at 0000000085f49266.
00000400:00000010:0.0:1787866226.326929:0:69374:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000005fef290b.
00000400:00000010:0.0:1787866226.326969:0:69374:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000066d6d332.
00000400:00000010:0.0:1787866226.326998:0:69374:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000af1cd23b.
00000400:00000200:0.0:1787866226.327008:0:69374:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.203.58@tcp
00000400:00000200:0.0:1787866226.327030:0:69374:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR:  192.168.203.58@tcp local destination
00000400:00000200:0.0:1787866226.327038:0:69374:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.58@tcp
00000400:00000200:0.0:1787866226.327041:0:69374:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [24:24:256:4294967295] 192.168.203.58@tcp LPNI seq info [7:7:8:4294967295]
00000400:00000200:0.0:1787866226.327048:0:69374:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:0.0:1787866226.327055:0:69374:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:<?>) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : GET try# 0
00000800:00000200:0.0:1787866226.327061:0:69374:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.203.58@tcp
00000800:00000010:0.0:1787866226.327066:0:69374:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000049f786d1.
00000800:00000200:0.0:1787866226.327074:0:69374:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ff36d1a3] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:0.0:1787866226.327084:0:69374:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000200:0.0:1787866226.327088:0:69374:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000049f786d1 type 2, nob 96 niov 1 nkiov 0
00000400:00000200:0.0:1787866226.327107:0:69374:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.203.58@tcp added to recovery queue. ping count: 3 next ping: 1444 last alive: 1428 health: 0
00000400:00000200:0.0:1787866226.327133:0:69384:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 3
00000400:00000200:0.0:1787866226.327138:0:69384:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.203.58@tcp(192.168.203.58@tcp)->dst 192.168.203.58@tcp: GET simulate health error: REMOTE_ERROR
00000800:02000400:0.0:1787866226.327141:0:69384:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.203.58: rc = -22
00000800:00000200:0.0:1787866226.327144:0:69384:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [000000008fe39939] Error -22 on write to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000010:0.0:1787866226.327148:0:69384:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000049f786d1.
00000400:00000200:0.0:1787866226.327155:0:69384:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787866226.327159:0:69384:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: GET: REMOTE_ERROR
00000400:00000200:0.0:1787866226.327165:0:69384:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5
00000400:00000200:0.0:1787866226.327168:0:69384:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.203.58@tcp recovery message sent unsuccessfully:-5
00000400:00000200:0.0:1787866226.327172:0:69384:0:(lib-move.c:4104:lnet_handle_recovery_reply()) peer NI (192.168.203.58@tcp) recovery failed with -5
00000400:00000200:2.0:1787866226.333294:0:69705:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:2.0:1787866226.333307:0:69705:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000f579002a.
00000400:00000200:2.0:1787866226.333321:0:69705:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:2.0:1787866226.333341:0:69705:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 000000003c4758a4.
00000400:00000200:2.0:1787866226.333349:0:69705:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:2.0:1787866226.333354:0:69705:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 000000003c4758a4.
00000400:00000010:2.0:1787866226.333363:0:69705:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000f579002a.
00000400:00000200:0.0:1787866227.348520:0:69374:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000066d6d332
00000400:00000010:0.0:1787866227.348534:0:69374:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 0000000066d6d332.
00000400:00000200:0.0:1787866227.348545:0:69374:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0
00000400:00000200:0.0:1787866227.348553:0:69374:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.203.58@tcp recovery ping unlinked
00000400:00000010:0.0:1787866227.348561:0:69374:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000085f49266.
00000400:00000010:0.0:1787866227.348567:0:69374:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000005fef290b.
00000400:00000200:0.0:1787866227.427981:0:69713:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:0.0:1787866227.427990:0:69713:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000008cf6d149.
00000400:00000200:0.0:1787866227.428006:0:69713:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:0.0:1787866227.428023:0:69713:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 000000000f65dfed.
00000400:00000200:0.0:1787866227.428030:0:69713:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:0.0:1787866227.428034:0:69713:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 000000000f65dfed.
00000400:00000010:0.0:1787866227.428041:0:69713:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000008cf6d149.
00000400:00000200:2.0:1787866228.525521:0:69721:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:2.0:1787866228.525530:0:69721:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000003c4758a4.
00000400:00000200:2.0:1787866228.525540:0:69721:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:2.0:1787866228.525560:0:69721:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000e4bb642e.
00000400:00000200:2.0:1787866228.525567:0:69721:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:2.0:1787866228.525961:0:69721:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000e4bb642e.
00000400:00000010:2.0:1787866228.525972:0:69721:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000003c4758a4.
00000400:00000200:2.0:1787866229.618257:0:69729:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:2.0:1787866229.618267:0:69729:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000003c4758a4.
00000400:00000200:2.0:1787866229.618282:0:69729:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:2.0:1787866229.618296:0:69729:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000e4bb642e.
00000400:00000200:2.0:1787866229.618303:0:69729:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:2.0:1787866229.618307:0:69729:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000e4bb642e.
00000400:00000010:2.0:1787866229.618316:0:69729:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000003c4758a4.
00000400:00000200:1.0:1787866230.707439:0:69737:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:1.0:1787866230.707449:0:69737:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000001b11e90c.
00000400:00000200:1.0:1787866230.707458:0:69737:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:1.0:1787866230.707477:0:69737:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000769b4e6d.
00000400:00000200:1.0:1787866230.707484:0:69737:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:1.0:1787866230.707488:0:69737:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000769b4e6d.
00000400:00000010:1.0:1787866230.707496:0:69737:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000001b11e90c.
00000400:00000200:1.0:1787866231.793209:0:69746:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:1.0:1787866231.793219:0:69746:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000001b11e90c.
00000400:00000200:1.0:1787866231.793229:0:69746:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:1.0:1787866231.793308:0:69746:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000769b4e6d.
00000400:00000200:1.0:1787866231.793375:0:69746:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:1.0:1787866231.793384:0:69746:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000769b4e6d.
00000400:00000010:1.0:1787866231.793390:0:69746:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000001b11e90c.
00000400:00000200:0.0:1787866232.868082:0:69754:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:0.0:1787866232.868092:0:69754:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 000000009d88af48.
00000400:00000200:0.0:1787866232.868102:0:69754:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:0.0:1787866232.868130:0:69754:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 000000008c3cd629.
00000400:00000200:0.0:1787866232.868138:0:69754:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:0.0:1787866232.868143:0:69754:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 000000008c3cd629.
00000400:00000010:0.0:1787866232.869032:0:69754:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 000000009d88af48.
00000400:00000200:1.0:1787866234.004740:0:69762:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:1.0:1787866234.004753:0:69762:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000769b4e6d.
00000400:00000200:1.0:1787866234.004765:0:69762:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:1.0:1787866234.004787:0:69762:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 000000009a8f11e0.
00000400:00000200:1.0:1787866234.004795:0:69762:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:1.0:1787866234.004800:0:69762:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 000000009a8f11e0.
00000400:00000010:1.0:1787866234.004811:0:69762:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000769b4e6d.
00000400:00000010:0.0:1787866234.516400:0:69374:0:(lib-move.c:3721:lnet_recover_peer_nis()) alloc '(ev_info)': 24 at 000000001b9612e5.
00000400:00000010:0.0:1787866234.516439:0:69374:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004d529970.
00000400:00000010:0.0:1787866234.516448:0:69374:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000066d6d332.
00000400:00000010:0.0:1787866234.516460:0:69374:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000e5d449cc.
00000400:00000200:0.0:1787866234.516471:0:69374:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.203.58@tcp
00000400:00000200:0.0:1787866234.516489:0:69374:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR:  192.168.203.58@tcp local destination
00000400:00000200:0.0:1787866234.516498:0:69374:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.58@tcp
00000400:00000200:0.0:1787866234.516503:0:69374:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [25:25:256:4294967295] 192.168.203.58@tcp LPNI seq info [8:8:8:4294967295]
00000400:00000200:0.0:1787866234.516510:0:69374:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:0.0:1787866234.516518:0:69374:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:<?>) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : GET try# 0
00000800:00000200:0.0:1787866234.516524:0:69374:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.203.58@tcp
00000800:00000010:0.0:1787866234.516531:0:69374:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000049f786d1.
00000800:00000200:0.0:1787866234.516539:0:69374:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ff36d1a3] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:0.0:1787866234.516549:0:69374:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000200:0.0:1787866234.516555:0:69374:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000049f786d1 type 2, nob 96 niov 1 nkiov 0
00000400:00000200:0.0:1787866234.516593:0:69374:0:(peer.c:4749:lnet_peer_ni_add_to_recoveryq_locked()) lpni 192.168.203.58@tcp aged out last alive 1428
00000400:00000200:2.0:1787866234.516763:0:69386:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 4
00000400:00000200:2.0:1787866234.516771:0:69386:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.203.58@tcp(192.168.203.58@tcp)->dst 192.168.203.58@tcp: GET simulate health error: REMOTE_ERROR
00000800:02000400:2.0:1787866234.516775:0:69386:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.203.58: rc = -22
00000800:00000200:2.0:1787866234.516780:0:69386:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [0000000085e71ae7] Error -22 on write to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000010:2.0:1787866234.516785:0:69386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000049f786d1.
00000400:00000200:2.0:1787866234.516792:0:69386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787866234.516796:0:69386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: GET: REMOTE_ERROR
00000400:00000200:2.0:1787866234.516803:0:69386:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5
00000400:00000200:2.0:1787866234.516806:0:69386:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.203.58@tcp recovery message sent unsuccessfully:-5
00000400:00000200:2.0:1787866234.516810:0:69386:0:(lib-move.c:4104:lnet_handle_recovery_reply()) peer NI (192.168.203.58@tcp) recovery failed with -5
00000400:00000200:3.0:1787866235.100304:0:69770:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:3.0:1787866235.100317:0:69770:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 0000000029538d91.
00000400:00000010:3.0:1787866235.100327:0:69770:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 0000000029538d91.
00000001:02000400:2.0:1787866235.179591:0:69772:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p
00000400:00000200:1.0:1787866235.224955:0:69773:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:1.0:1787866235.224967:0:69773:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000769b4e6d.
00000400:00000010:1.0:1787866235.224976:0:69773:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000769b4e6d.
00000400:00000010:3.0:1787866235.308365:0:69779:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 00000000df680292.
00000400:00000010:3.0:1787866235.308887:0:69779:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 00000000df680292.
00000400:00000010:3.0:1787866235.403358:0:69784:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 00000000df680292.
00000400:00000010:3.0:1787866235.404915:0:69784:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 00000000df680292.
00000001:02000400:3.0:1787866235.430199:0:69787:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop del -a
00000400:00000200:3.0:1787866235.455622:0:69788:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866235.455755:0:69788:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866235.455763:0:69788:0:(api-ni.c:3207:nla_strnid()) <?> -> <?>
00000400:00000200:3.0:1787866235.455765:0:69788:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866235.455766:0:69788:0:(api-ni.c:3207:nla_strnid()) <?> -> <?>
00000400:00000200:3.0:1787866235.455770:0:69788:0:(net_fault.c:254:lnet_drop_rule_del()) src <?> dst <?>
00000400:00000200:3.0:1787866235.455967:0:69788:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp->dst: 255.255.255.255@tcp (1/1, 0)
00000400:00000010:3.0:1787866235.455973:0:69788:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 000000008b8310b3.
00000400:00000200:3.0:1787866235.455982:0:69788:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp1->dst: 255.255.255.255@tcp1 (1/1, 0)
00000400:00000010:3.0:1787866235.455984:0:69788:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 000000002eacacef.
00000400:00000200:3.0:1787866235.455987:0:69788:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp->dst: 255.255.255.255@tcp (1/1, 0)
00000400:00000010:3.0:1787866235.455989:0:69788:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 000000005df545ae.
00000400:00000200:3.0:1787866235.455993:0:69788:0:(net_fault.c:272:lnet_drop_rule_del()) Remove drop rule: src 255.255.255.255@tcp1->dst: 255.255.255.255@tcp1 (1/1, 0)
00000400:00000010:3.0:1787866235.455995:0:69788:0:(net_fault.c:275:lnet_drop_rule_del()) kfreed 'rule': 312 at 00000000159023a2.
00000001:02000400:2.0:1787866235.477526:0:69789:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 0
00000001:02000400:3.0:1787866235.544052:0:69791:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --nid 192.168.203.58@tcp --health 500
00000400:00000200:2.0:1787866235.573239:0:69792:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.203.58@tcp added to recovery queue. ping count: 0 next ping: 1446 last alive: 1428 health: 500
00000400:00000200:0.0:1787866235.667569:0:69798:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:0.0:1787866235.667584:0:69798:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000442414b0.
00000400:00000200:0.0:1787866235.667598:0:69798:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:0.0:1787866235.669709:0:69798:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 000000007fc9ad76.
00000400:00000200:0.0:1787866235.669737:0:69798:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:0.0:1787866235.669746:0:69798:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 000000007fc9ad76.
00000400:00000010:0.0:1787866235.669765:0:69798:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000442414b0.
00000001:02000400:3.0:1787866235.726499:0:69800:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p
00000400:00000200:2.0:1787866235.772805:0:69801:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:2.0:1787866235.772816:0:69801:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000aab07357.
00000400:00000200:2.0:1787866235.772830:0:69801:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:2.0:1787866235.772880:0:69801:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 000000003f0bdf4c.
00000400:00000200:2.0:1787866235.772889:0:69801:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:2.0:1787866235.772895:0:69801:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 000000003f0bdf4c.
00000400:00000010:2.0:1787866235.772923:0:69801:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000aab07357.
00000400:00000200:2.0:1787866235.853035:0:69807:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:2.0:1787866235.853047:0:69807:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000aab07357.
00000400:00000200:2.0:1787866235.853061:0:69807:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:2.0:1787866235.853083:0:69807:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 000000003f0bdf4c.
00000400:00000200:2.0:1787866235.853098:0:69807:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:2.0:1787866235.853102:0:69807:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 000000003f0bdf4c.
00000400:00000010:2.0:1787866235.853112:0:69807:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000aab07357.
00000400:00000200:0.0:1787866236.563415:0:69374:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000066d6d332
00000400:00000010:0.0:1787866236.563432:0:69374:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 0000000066d6d332.
00000400:00000200:0.0:1787866236.563442:0:69374:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0
00000400:00000200:0.0:1787866236.563451:0:69374:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.203.58@tcp recovery ping unlinked
00000400:00000010:0.0:1787866236.563457:0:69374:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000001b9612e5.
00000400:00000010:0.0:1787866236.563463:0:69374:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004d529970.
00000400:00000010:0.0:1787866236.563475:0:69374:0:(lib-move.c:3721:lnet_recover_peer_nis()) alloc '(ev_info)': 24 at 000000001b9612e5.
00000400:00000010:0.0:1787866236.563483:0:69374:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004d529970.
00000400:00000010:0.0:1787866236.563488:0:69374:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000066d6d332.
00000400:00000010:0.0:1787866236.563496:0:69374:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000094cf90b.
00000400:00000200:0.0:1787866236.563500:0:69374:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.203.58@tcp
00000400:00000200:0.0:1787866236.563514:0:69374:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR:  192.168.203.58@tcp local destination
00000400:00000200:0.0:1787866236.563522:0:69374:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.58@tcp
00000400:00000200:0.0:1787866236.563526:0:69374:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [26:26:256:4294967295] 192.168.203.58@tcp LPNI seq info [9:9:8:4294967295]
00000400:00000200:0.0:1787866236.563533:0:69374:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:0.0:1787866236.563540:0:69374:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:<?>) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : GET try# 0
00000800:00000200:0.0:1787866236.563545:0:69374:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.203.58@tcp
00000800:00000010:0.0:1787866236.563550:0:69374:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000d36d337d.
00000800:00000200:0.0:1787866236.563558:0:69374:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ff36d1a3] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:0.0:1787866236.563567:0:69374:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000200:0.0:1787866236.563572:0:69374:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000d36d337d type 2, nob 96 niov 1 nkiov 0
00000400:00000200:0.0:1787866236.563599:0:69374:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.203.58@tcp added to recovery queue. ping count: 1 next ping: 1448 last alive: 1428 health: 500
00000800:00000200:3.0:1787866236.564688:0:69385:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1787866236.564700:0:69385:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000d36d337d.
00000400:00000200:3.0:1787866236.564726:0:69385:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1787866236.564734:0:69385:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: GET: OK
00000400:00000200:3.0:1787866236.564742:0:69385:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0
00000400:00000200:3.0:1787866236.564745:0:69385:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.203.58@tcp recovery message sent successfully:0
00000400:00000200:3.0:1787866236.564789:0:69385:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : GET - for me
00000400:00000200:3.0:1787866236.564826:0:69385:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.58@tcp of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:3.0:1787866236.564871:0:69385:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.203.58@tcp of length 64/272 into md 0xd [1] + 0
00000400:00000200:3.0:1787866236.564885:0:69385:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.203.58@tcp to MR:  192.168.203.58@tcp local destination
00000400:00000200:3.0:1787866236.564890:0:69385:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [27:27:256:4294967295] 192.168.203.58@tcp LPNI seq info [10:10:8:4294967295]
00000400:00000200:3.0:1787866236.564901:0:69385:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) 192.168.203.58@tcp : REPLY try# 0
00000800:00000200:3.0:1787866236.564906:0:69385:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.203.58@tcp
00000800:00000010:3.0:1787866236.564912:0:69385:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000087d0ec8d.
00000800:00000200:3.0:1787866236.564918:0:69385:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ff36d1a3] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:3.0:1787866236.564926:0:69385:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000200:3.0:1787866236.564929:0:69385:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000087d0ec8d type 3, nob 160 niov 1 nkiov 1
00000800:00000200:1.0:1787866236.565657:0:69384:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1787866236.565666:0:69384:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000087d0ec8d.
00000400:00000200:1.0:1787866236.565673:0:69384:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1787866236.565679:0:69384:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: REPLY: OK
00000400:00000200:1.0:1787866236.565702:0:69384:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : REPLY - for me
00000400:00000200:1.0:1787866236.565713:0:69384:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.203.58@tcp: Reply from 12345-192.168.203.58@tcp of length 64/64 into md 0x2d
00000400:00000200:1.0:1787866236.565722:0:69384:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1787866236.565725:0:69384:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: REPLY: OK
00000400:00000200:1.0:1787866236.565729:0:69384:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000066d6d332
00000400:00000010:1.0:1787866236.565732:0:69384:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 0000000066d6d332.
00000400:00000200:1.0:1787866236.565736:0:69384:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0
00000400:00000010:1.0:1787866236.565740:0:69384:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000001b9612e5.
00000400:00000010:1.0:1787866236.565744:0:69384:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004d529970.
00000400:00000200:1.0:1787866237.008264:0:69815:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:1.0:1787866237.008274:0:69815:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 0000000096161dcb.
00000400:00000200:1.0:1787866237.008287:0:69815:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:1.0:1787866237.008307:0:69815:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 000000004ce5a06f.
00000400:00000200:1.0:1787866237.008315:0:69815:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:1.0:1787866237.008319:0:69815:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 000000004ce5a06f.
00000400:00000010:1.0:1787866237.008328:0:69815:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 0000000096161dcb.
00000400:00000200:3.0:1787866238.117223:0:69824:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:3.0:1787866238.117236:0:69824:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000c58aff75.
00000400:00000200:3.0:1787866238.117250:0:69824:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:3.0:1787866238.117271:0:69824:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000f71bad92.
00000400:00000200:3.0:1787866238.117279:0:69824:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:3.0:1787866238.117283:0:69824:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000f71bad92.
00000400:00000010:3.0:1787866238.117293:0:69824:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000c58aff75.
00000400:00000010:0.0:1787866238.612058:0:69374:0:(lib-move.c:3721:lnet_recover_peer_nis()) alloc '(ev_info)': 24 at 00000000bd4784bd.
00000400:00000010:0.0:1787866238.612078:0:69374:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000008e441545.
00000400:00000010:0.0:1787866238.612085:0:69374:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f517fc46.
00000400:00000010:0.0:1787866238.612095:0:69374:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000026e8d34f.
00000400:00000200:0.0:1787866238.612105:0:69374:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.203.58@tcp
00000400:00000200:0.0:1787866238.612147:0:69374:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR:  192.168.203.58@tcp local destination
00000400:00000200:0.0:1787866238.612155:0:69374:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.58@tcp
00000400:00000200:0.0:1787866238.612159:0:69374:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [28:28:256:4294967295] 192.168.203.58@tcp LPNI seq info [11:11:8:4294967295]
00000400:00000200:0.0:1787866238.612167:0:69374:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:0.0:1787866238.612176:0:69374:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:<?>) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : GET try# 0
00000800:00000200:0.0:1787866238.612182:0:69374:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.203.58@tcp
00000800:00000010:0.0:1787866238.612187:0:69374:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f46aa85a.
00000800:00000200:0.0:1787866238.612195:0:69374:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ff36d1a3] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:0.0:1787866238.612204:0:69374:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000200:0.0:1787866238.612210:0:69374:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f46aa85a type 2, nob 96 niov 1 nkiov 0
00000400:00000200:0.0:1787866238.612240:0:69374:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.203.58@tcp added to recovery queue. ping count: 1 next ping: 1450 last alive: 1446 health: 700
00000800:00000200:1.0:1787866238.613804:0:69384:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1787866238.613813:0:69384:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000f46aa85a.
00000400:00000200:1.0:1787866238.613823:0:69384:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1787866238.613871:0:69384:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: GET: OK
00000400:00000200:1.0:1787866238.613879:0:69384:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0
00000400:00000200:1.0:1787866238.613883:0:69384:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.203.58@tcp recovery message sent successfully:0
00000400:00000200:1.0:1787866238.613922:0:69384:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : GET - for me
00000400:00000200:1.0:1787866238.613934:0:69384:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.58@tcp of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:1.0:1787866238.613943:0:69384:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.203.58@tcp of length 64/272 into md 0xd [1] + 0
00000400:00000200:1.0:1787866238.613963:0:69384:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.203.58@tcp to MR:  192.168.203.58@tcp local destination
00000400:00000200:1.0:1787866238.613969:0:69384:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [29:29:256:4294967295] 192.168.203.58@tcp LPNI seq info [12:12:8:4294967295]
00000400:00000200:1.0:1787866238.613980:0:69384:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) 192.168.203.58@tcp : REPLY try# 0
00000800:00000200:1.0:1787866238.613985:0:69384:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.203.58@tcp
00000800:00000010:1.0:1787866238.613990:0:69384:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000009dab4a5c.
00000800:00000200:1.0:1787866238.613996:0:69384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ff36d1a3] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:1.0:1787866238.614004:0:69384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000200:1.0:1787866238.614008:0:69384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000009dab4a5c type 3, nob 160 niov 1 nkiov 1
00000800:00000200:1.0:1787866238.614070:0:69384:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1787866238.614073:0:69384:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000009dab4a5c.
00000400:00000200:1.0:1787866238.614075:0:69384:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1787866238.614079:0:69384:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: REPLY: OK
00000400:00000200:1.0:1787866238.614093:0:69384:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : REPLY - for me
00000400:00000200:1.0:1787866238.614100:0:69384:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.203.58@tcp: Reply from 12345-192.168.203.58@tcp of length 64/64 into md 0x31
00000400:00000200:1.0:1787866238.614108:0:69384:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1787866238.614121:0:69384:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: REPLY: OK
00000400:00000200:1.0:1787866238.614127:0:69384:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f517fc46
00000400:00000010:1.0:1787866238.614129:0:69384:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000f517fc46.
00000400:00000200:1.0:1787866238.614133:0:69384:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0
00000400:00000010:1.0:1787866238.614136:0:69384:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000bd4784bd.
00000400:00000010:1.0:1787866238.614155:0:69384:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000008e441545.
00000400:00000200:2.0:1787866239.248772:0:69832:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:2.0:1787866239.248784:0:69832:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 0000000006ca9eb8.
00000400:00000200:2.0:1787866239.248797:0:69832:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:2.0:1787866239.248827:0:69832:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000f9cb8dba.
00000400:00000200:2.0:1787866239.248856:0:69832:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:2.0:1787866239.248863:0:69832:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000f9cb8dba.
00000400:00000010:2.0:1787866239.248870:0:69832:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 0000000006ca9eb8.
00000400:00000200:3.0:1787866240.356687:0:69840:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:3.0:1787866240.356701:0:69840:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000f71bad92.
00000400:00000200:3.0:1787866240.356716:0:69840:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:3.0:1787866240.356738:0:69840:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 00000000be2f6e10.
00000400:00000200:3.0:1787866240.356747:0:69840:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:3.0:1787866240.356751:0:69840:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 00000000be2f6e10.
00000400:00000010:3.0:1787866240.356763:0:69840:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000f71bad92.
00000400:00000010:0.0:1787866240.658995:0:69374:0:(lib-move.c:3721:lnet_recover_peer_nis()) alloc '(ev_info)': 24 at 0000000085f49266.
00000400:00000010:0.0:1787866240.659016:0:69374:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000005fef290b.
00000400:00000010:0.0:1787866240.659024:0:69374:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000074d487e2.
00000400:00000010:0.0:1787866240.659037:0:69374:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000007c34e004.
00000400:00000200:0.0:1787866240.659048:0:69374:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.203.58@tcp
00000400:00000200:0.0:1787866240.659423:0:69374:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR:  192.168.203.58@tcp local destination
00000400:00000200:0.0:1787866240.659432:0:69374:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.58@tcp
00000400:00000200:0.0:1787866240.659436:0:69374:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [30:30:256:4294967295] 192.168.203.58@tcp LPNI seq info [13:13:8:4294967295]
00000400:00000200:0.0:1787866240.659445:0:69374:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:0.0:1787866240.659452:0:69374:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:<?>) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : GET try# 0
00000800:00000200:0.0:1787866240.659458:0:69374:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.203.58@tcp
00000800:00000010:0.0:1787866240.659465:0:69374:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f4dc10dd.
00000800:00000200:0.0:1787866240.659473:0:69374:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ff36d1a3] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:0.0:1787866240.659483:0:69374:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000200:0.0:1787866240.659488:0:69374:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f4dc10dd type 2, nob 96 niov 1 nkiov 0
00000400:00000200:0.0:1787866240.659574:0:69374:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.203.58@tcp added to recovery queue. ping count: 1 next ping: 1452 last alive: 1448 health: 900
00000800:00000200:1.0:1787866240.663221:0:69384:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1787866240.663234:0:69384:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000f4dc10dd.
00000400:00000200:1.0:1787866240.663247:0:69384:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1787866240.663258:0:69384:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: GET: OK
00000400:00000200:1.0:1787866240.663266:0:69384:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0
00000400:00000200:1.0:1787866240.663270:0:69384:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.203.58@tcp recovery message sent successfully:0
00000400:00000200:1.0:1787866240.663559:0:69384:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : GET - for me
00000400:00000200:1.0:1787866240.663577:0:69384:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.58@tcp of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:1.0:1787866240.663587:0:69384:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.203.58@tcp of length 64/272 into md 0xd [1] + 0
00000400:00000200:1.0:1787866240.663603:0:69384:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.203.58@tcp to MR:  192.168.203.58@tcp local destination
00000400:00000200:1.0:1787866240.663609:0:69384:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [31:31:256:4294967295] 192.168.203.58@tcp LPNI seq info [14:14:8:4294967295]
00000400:00000200:1.0:1787866240.663621:0:69384:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) 192.168.203.58@tcp : REPLY try# 0
00000800:00000200:1.0:1787866240.663626:0:69384:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.203.58@tcp
00000800:00000010:1.0:1787866240.663633:0:69384:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000009dab4a5c.
00000800:00000200:1.0:1787866240.663640:0:69384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ff36d1a3] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:1.0:1787866240.663649:0:69384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000200:1.0:1787866240.663653:0:69384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000009dab4a5c type 3, nob 160 niov 1 nkiov 1
00000800:00000200:2.0:1787866240.666737:0:69386:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1787866240.666750:0:69386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000009dab4a5c.
00000400:00000200:2.0:1787866240.666758:0:69386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1787866240.666769:0:69386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: REPLY: OK
00000400:00000200:2.0:1787866240.666809:0:69386:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : REPLY - for me
00000400:00000200:2.0:1787866240.666827:0:69386:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.203.58@tcp: Reply from 12345-192.168.203.58@tcp of length 64/64 into md 0x35
00000400:00000200:2.0:1787866240.666860:0:69386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1787866240.666863:0:69386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: REPLY: OK
00000400:00000200:2.0:1787866240.666868:0:69386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000074d487e2
00000400:00000010:2.0:1787866240.666871:0:69386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 0000000074d487e2.
00000400:00000200:2.0:1787866240.666875:0:69386:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0
00000400:00000010:2.0:1787866240.666880:0:69386:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000085f49266.
00000400:00000010:2.0:1787866240.666885:0:69386:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000005fef290b.
00000400:00000200:1.0:1787866241.501416:0:69848:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:1.0:1787866241.501429:0:69848:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 0000000030468a31.
00000400:00000200:1.0:1787866241.501443:0:69848:0:(api-ni.c:9583:lnet_debug_recovery_show_start()) nid: 192.168.203.58@tcp
00000400:00000010:1.0:1787866241.501464:0:69848:0:(api-ni.c:9645:lnet_debug_recovery_show_dump()) alloc '(keys)': 40 at 0000000011487433.
00000400:00000200:1.0:1787866241.501473:0:69848:0:(api-ni.c:9698:lnet_debug_recovery_show_dump()) nid: 192.168.203.58@tcp
00000400:00000010:1.0:1787866241.501478:0:69848:0:(api-ni.c:9496:lnet_debug_recovery_show_done()) kfreed 'drlist->lgdrl_keys': 40 at 0000000011487433.
00000400:00000010:1.0:1787866241.501486:0:69848:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 0000000030468a31.
00000400:00000200:1.0:1787866242.640071:0:69856:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:1.0:1787866242.640094:0:69856:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 0000000030468a31.
00000400:00000010:1.0:1787866242.640104:0:69856:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 0000000030468a31.
00000001:02000400:1.0:1787866242.708617:0:69858:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl debug recovery -p
00000400:00000200:3.0:1787866242.748568:0:69859:0:(api-ni.c:9546:lnet_debug_recovery_show_start()) Got queue_type: 1
00000400:00000010:3.0:1787866242.748582:0:69859:0:(api-ni.c:9548:lnet_debug_recovery_show_start()) alloc '(drlist)': 40 at 00000000be2f6e10.
00000400:00000010:3.0:1787866242.748593:0:69859:0:(api-ni.c:9499:lnet_debug_recovery_show_done()) kfreed 'drlist': 40 at 00000000be2f6e10.
00000001:02000400:2.0:1787866242.832618:0:69862:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
00000400:00000200:1.0:1787866242.896327:0:69863:0:(net_fault.c:254:lnet_drop_rule_del()) src <?> dst <?>
00000400:00000200:2.0:1787866242.896446:0:69375:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: -108
00000400:00000200:2.0:1787866242.896453:0:69375:0:(peer.c:4385:lnet_peer_discovery()) stopping
00000400:00000200:2.0:1787866242.896467:0:69375:0:(peer.c:4431:lnet_peer_discovery()) stopped
00000400:00000200:1.0:1787866242.899438:0:69863:0:(peer.c:4488:lnet_peer_discovery_stop()) discovery stopped
00000400:00000010:1.0:1787866242.899932:0:69863:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 0000000089805c47.
00000400:00000010:1.0:1787866242.899942:0:69863:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 00000000d47bc20e.
00000400:00000010:1.0:1787866242.899943:0:69863:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 00000000eefdcd83.
00000400:00000010:1.0:1787866242.899946:0:69863:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 000000008942af4d.
00000400:00000010:1.0:1787866242.899948:0:69863:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 00000000af1cd23b.
00000400:00000010:1.0:1787866242.899950:0:69863:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 00000000e5d449cc.
00000400:00000010:1.0:1787866242.899952:0:69863:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 00000000094cf90b.
00000400:00000010:1.0:1787866242.899954:0:69863:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 0000000026e8d34f.
00000400:00000010:1.0:1787866242.899956:0:69863:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at 000000007c34e004.
00000400:00000010:1.0:1787866242.899959:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000d9b6c465.
00000400:00000010:1.0:1787866242.899962:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000008876f365.
00000400:00000010:1.0:1787866242.899969:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f95cda80.
00000400:00000010:1.0:1787866242.899971:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000023cd7e93.
00000400:00000010:1.0:1787866242.899982:0:69863:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000094649d34.
00000400:00000200:1.0:1787866242.899987:0:69863:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e4d53631
00000400:00000010:1.0:1787866242.899990:0:69863:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000e4d53631.
00000400:00000200:1.0:1787866242.899996:0:69863:0:(api-ni.c:2393:lnet_push_target_event_handler()) type 6 status 0 unlinked 1
00000400:00000010:1.0:1787866242.900001:0:69863:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000004ece727.
00000400:00000400:1.0:1787866242.900006:0:69863:0:(lib-ptl.c:964:lnet_clear_lazy_portal()) Active lazy portal 0 on exit
00000400:00000010:1.0:1787866242.913365:0:69863:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000000061a5be.
00000400:00000200:1.0:1787866242.913397:0:69863:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000000c004c47
00000400:00000010:1.0:1787866242.913400:0:69863:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000000c004c47.
00000400:00000010:0.0:1787866242.913482:0:69863:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 73 at 000000009dea2467.
00000400:00000200:3.0:1787866242.913962:0:69387:0:(acceptor.c:659:lnet_acceptor()) Acceptor stopping
00000400:00000200:0.0:1787866242.914217:0:69863:0:(router.c:792:lnet_del_route()) Del route: net <255:65535> : gw <?>
00000400:00000200:0.0:1787866242.914255:0:69863:0:(lo.c:49:lolnd_shutdown()) shutdown
00000400:00000010:0.0:1787866242.914259:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000027b2bdf8.
00000400:00000010:0.0:1787866242.914263:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000005586a200.
00000400:00000010:0.0:1787866242.914266:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000054555ad8.
00000400:00000010:0.0:1787866242.914268:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000ef2cd2ea.
00000400:00000010:0.0:1787866242.914271:0:69863:0:(config.c:293:lnet_ni_free()) kfreed 'ni': 552 at 00000000b026a959.
00000400:00000010:0.0:1787866242.914284:0:69863:0:(config.c:329:lnet_net_free()) kfreed 'net': 176 at 00000000481f47bd.
00000800:00000200:0.1:1787866242.914302:0:69863:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 2, ksnr_connected a ksnr_max_conns 1
00000800:00000200:0.1:1787866242.914315:0:69863:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 3, ksnr_connected 2 ksnr_max_conns 1
00000800:00000200:0.1:1787866242.914318:0:69863:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 1, ksnr_connected 2 ksnr_max_conns 1
00000800:00000200:0.1:1787866242.914319:0:69863:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 1, ksnr_connected 0 ksnr_max_conns 1
00000800:00000010:0.1:1787866242.914323:0:69863:0:(socklnd.c:126:ksocknal_destroy_conn_cb()) kfreed 'conn_cb': 192 at 00000000e80b4a01.
00000400:00020000:0.0:1787866242.914329:0:69863:0:(acceptor.c:252:lnet_acceptor_remove_socket()) Interface ens2 not found
00000800:00000200:3.0:1787866242.916917:0:69383:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 000000008d8cdc37
00000800:00000010:3.0:1787866242.916927:0:69383:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 000000008d8cdc37.
00000800:00000200:3.0:1787866242.916931:0:69383:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 000000008fe39939
00000800:00000010:3.0:1787866242.916933:0:69383:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 000000008fe39939.
00000800:00000200:3.0:1787866242.916963:0:69383:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 00000000eec38694
00000800:00000010:3.0:1787866242.916965:0:69383:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 00000000eec38694.
00000800:00000200:3.0:1787866242.916968:0:69383:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection 0000000085e71ae7
00000800:00000200:3.0:1787866242.916977:0:69383:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.203.58@tcp 00000000ff36d1a3 deleted
00000800:00000010:3.0:1787866242.916980:0:69383:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 00000000ff36d1a3.
00000800:00000010:3.0:1787866242.916983:0:69383:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at 0000000085e71ae7.
00000800:00000010:3.0:1787866242.937395:0:69863:0:(socklnd.c:2498:ksocknal_shutdown()) kfreed 'net': 208 at 000000001b022c72.
00000400:02000000:3.0:1787866242.937426:0:69863:0:(api-ni.c:2563:lnet_clear_zombies_nis_locked()) Removed LNI 192.168.203.58@tcp
00000400:00000010:3.0:1787866242.939327:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000b90a4287.
00000400:00000010:3.0:1787866242.939344:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000006435fc69.
00000400:00000010:3.0:1787866242.939348:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000075b49a8d.
00000400:00000010:3.0:1787866242.939349:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000cd87e7a3.
00000400:00000010:3.0:1787866242.939353:0:69863:0:(config.c:286:lnet_ni_free()) kfreed 'ni->ni_interface': 5 at 00000000ee9673b3.
00000400:00000010:3.0:1787866242.939356:0:69863:0:(config.c:293:lnet_ni_free()) kfreed 'ni': 552 at 00000000dd651a5c.
00000400:00000200:3.0:1787866242.939379:0:69863:0:(peer.c:620:lnet_peer_ni_clr_local_net_locked()) 0000000060c33e75 nid 192.168.203.58@tcp detached from local net
00000400:00000010:3.0:1787866242.939384:0:69863:0:(config.c:329:lnet_net_free()) kfreed 'net': 176 at 000000004ebeb1be.
00000400:00020000:3.0:1787866242.939411:0:69863:0:(acceptor.c:252:lnet_acceptor_remove_socket()) Interface ens2 not found
00000800:00000010:3.0:1787866242.939415:0:69863:0:(socklnd.c:2498:ksocknal_shutdown()) kfreed 'net': 208 at 00000000e03769d4.
00000800:00000010:3.0:1787866242.939420:0:69863:0:(socklnd.c:2235:ksocknal_base_shutdown()) before NAL cleanup: kmem 418789
00000800:00000200:3.0:1787866242.939430:0:69863:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_GOING_DOWN, iface=lo ifindex 1 state 0 ns 00000000198b8817
00000800:00000200:3.0:1787866242.939433:0:69863:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_DOWN, iface=lo ifindex 1 state 0 ns 00000000198b8817
00000800:00000200:3.0:1787866242.939435:0:69863:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UNREGISTER, iface=lo ifindex 1 state 0 ns 00000000198b8817
00000800:00000200:3.0:1787866242.939437:0:69863:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_GOING_DOWN, iface=ens2 ifindex 2 state 6 ns 00000000198b8817
00000800:00000200:3.0:1787866242.939439:0:69863:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_DOWN, iface=ens2 ifindex 2 state 6 ns 00000000198b8817
00000800:00000200:3.0:1787866242.939467:0:69863:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UNREGISTER, iface=ens2 ifindex 2 state 6 ns 00000000198b8817
00000400:00000010:2.0:1787866242.951868:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 00000000891e7618.
00000400:00000010:2.0:1787866242.951881:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000006ef8a308.
00000800:00000010:2.0:1787866242.951885:0:69863:0:(socklnd.c:2296:ksocknal_base_shutdown()) after NAL cleanup: kmem 418637
00000400:02000000:2.0:1787866242.951898:0:69863:0:(api-ni.c:2563:lnet_clear_zombies_nis_locked()) Removed LNI 192.168.203.58@tcp1
00000400:00000010:2.0:1787866242.954816:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000000c2cb86.
00000400:00000010:2.0:1787866242.954875:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000e143976c.
00000400:00000010:2.0:1787866242.954897:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000066728e94.
00000400:00000010:2.0:1787866242.954899:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000442aa5e4.
00000400:00000010:2.0:1787866242.954903:0:69863:0:(config.c:286:lnet_ni_free()) kfreed 'ni->ni_interface': 5 at 00000000d0789159.
00000400:00000010:2.0:1787866242.954906:0:69863:0:(config.c:293:lnet_ni_free()) kfreed 'ni': 552 at 000000008aa2db0b.
00000400:00000200:2.0:1787866242.954926:0:69863:0:(peer.c:442:lnet_peer_del_locked()) peer 192.168.203.58@tcp1
00000400:00000200:2.0:1787866242.954932:0:69863:0:(peer.c:364:lnet_peer_detach_peer_ni_locked()) peer 192.168.203.58@tcp1 NID 192.168.203.58@tcp
00000400:00000200:2.0:1787866242.954935:0:69863:0:(peer.c:2392:lnet_destroy_peer_ni_locked()) 0000000060c33e75 nid 192.168.203.58@tcp
00000400:00000010:2.0:1787866242.954939:0:69863:0:(peer.c:2422:lnet_destroy_peer_ni_locked()) kfreed 'lpni': 480 at 0000000060c33e75.
00000400:00000200:2.0:1787866242.954944:0:69863:0:(peer.c:213:lnet_destroy_peer_net_locked()) 00000000548ae7a0 net tcp
00000400:00000010:2.0:1787866242.954946:0:69863:0:(peer.c:220:lnet_destroy_peer_net_locked()) kfreed 'lpn': 72 at 00000000548ae7a0.
00000400:00000200:2.0:1787866242.954950:0:69863:0:(peer.c:364:lnet_peer_detach_peer_ni_locked()) peer 192.168.203.58@tcp1 NID 192.168.203.58@tcp1
00000400:00000200:2.0:1787866242.954952:0:69863:0:(peer.c:2392:lnet_destroy_peer_ni_locked()) 000000008ee7702b nid 192.168.203.58@tcp1
00000400:00000010:2.0:1787866242.954954:0:69863:0:(peer.c:2422:lnet_destroy_peer_ni_locked()) kfreed 'lpni': 480 at 000000008ee7702b.
00000400:00000200:2.0:1787866242.954956:0:69863:0:(peer.c:213:lnet_destroy_peer_net_locked()) 0000000010a6e38f net tcp1
00000400:00000010:2.0:1787866242.954958:0:69863:0:(peer.c:220:lnet_destroy_peer_net_locked()) kfreed 'lpn': 72 at 0000000010a6e38f.
00000400:00000200:2.0:1787866242.954961:0:69863:0:(peer.c:274:lnet_destroy_peer_locked()) 00000000bd1acbb4 nid 192.168.203.58@tcp1
00000400:00000010:2.0:1787866242.954964:0:69863:0:(peer.c:304:lnet_destroy_peer_locked()) kfreed 'lp': 360 at 00000000bd1acbb4.
00000400:00000010:2.0:1787866242.954970:0:69863:0:(config.c:329:lnet_net_free()) kfreed 'net': 176 at 0000000060f85c02.
00000400:00000010:2.0:1787866242.954978:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000005b139ed2.
00000400:00000010:2.0:1787866242.954980:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000026f6c9c0.
00000400:00000010:2.0:1787866242.954989:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000066b6d98f.
00000400:00000010:2.0:1787866242.954992:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000007071892.
00000400:00000010:2.0:1787866242.954994:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000eb1b95d7.
00000400:00000010:2.0:1787866242.954996:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000050cc6800.
00000400:00000010:2.0:1787866242.955001:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f783d8b3.
00000400:00000010:2.0:1787866242.955003:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000280e6db6.
00000400:00000010:2.0:1787866242.955005:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000028069879.
00000400:00000010:2.0:1787866242.955007:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000317fbbc1.
00000400:00000010:2.0:1787866242.955010:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000b1b59a22.
00000400:00000010:2.0:1787866242.955012:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000bed862ec.
00000400:00000010:2.0:1787866242.955014:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000543069cf.
00000400:00000010:2.0:1787866242.955015:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000067e51d17.
00000400:00000010:2.0:1787866242.955019:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ec41b373.
00000400:00000010:2.0:1787866242.955028:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f3dfce7b.
00000400:00000010:2.0:1787866242.955029:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000007cbed98b.
00000400:00000010:2.0:1787866242.955031:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000000770e695.
00000400:00000010:2.0:1787866242.955034:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000002bcb5613.
00000400:00000010:2.0:1787866242.955036:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000006fab2734.
00000400:00000010:2.0:1787866242.955037:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002262e8a9.
00000400:00000010:2.0:1787866242.955039:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000005aa09076.
00000400:00000010:2.0:1787866242.955042:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000061615e9.
00000400:00000010:2.0:1787866242.955044:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000003872e290.
00000400:00000010:2.0:1787866242.955046:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000006bdf237.
00000400:00000010:2.0:1787866242.955047:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000c5f9be91.
00000400:00000010:2.0:1787866242.955050:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000a01446bf.
00000400:00000010:2.0:1787866242.955052:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000a6a5a280.
00000400:00000010:2.0:1787866242.955054:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000060a6f9d.
00000400:00000010:2.0:1787866242.955055:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000077764818.
00000400:00000010:2.0:1787866242.955059:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ce65c75a.
00000400:00000010:2.0:1787866242.955061:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f48cfbf4.
00000400:00000010:2.0:1787866242.955063:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000006c7f0227.
00000400:00000010:2.0:1787866242.955064:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000edc9be73.
00000400:00000010:2.0:1787866242.955067:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000a27db30f.
00000400:00000010:2.0:1787866242.955070:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000038e79002.
00000400:00000010:2.0:1787866242.955072:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000e945666a.
00000400:00000010:2.0:1787866242.955073:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000006e79233e.
00000400:00000010:2.0:1787866242.955076:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000cb480af2.
00000400:00000010:2.0:1787866242.955078:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000efa8010d.
00000400:00000010:2.0:1787866242.955079:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000003164c3ac.
00000400:00000010:2.0:1787866242.955092:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000557c67ca.
00000400:00000010:2.0:1787866242.955096:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000007434b255.
00000400:00000010:2.0:1787866242.955099:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000215b7ab4.
00000400:00000010:2.0:1787866242.955101:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000004c8208e0.
00000400:00000010:2.0:1787866242.955102:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000014e18e51.
00000400:00000010:2.0:1787866242.955106:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000eff1234f.
00000400:00000010:2.0:1787866242.955109:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000082f13997.
00000400:00000010:2.0:1787866242.955115:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000001c04d9f8.
00000400:00000010:2.0:1787866242.955117:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000c980c965.
00000400:00000010:2.0:1787866242.955120:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000000b3665f6.
00000400:00000010:2.0:1787866242.955123:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000c9c5a6f3.
00000400:00000010:2.0:1787866242.955124:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000001923fd6d.
00000400:00000010:2.0:1787866242.955125:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000005408f50.
00000400:00000010:2.0:1787866242.955129:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000003506a16b.
00000400:00000010:2.0:1787866242.955135:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000476ae7b8.
00000400:00000010:2.0:1787866242.955137:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002673251f.
00000400:00000010:2.0:1787866242.955138:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000001b28f015.
00000400:00000010:2.0:1787866242.955141:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000000ee9133.
00000400:00000010:2.0:1787866242.955145:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000cefc814e.
00000400:00000010:2.0:1787866242.955147:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000e621c0b1.
00000400:00000010:2.0:1787866242.955149:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000060507bb8.
00000400:00000010:2.0:1787866242.955152:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000e4747cb8.
00000400:00000010:2.0:1787866242.955154:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000082484639.
00000400:00000010:2.0:1787866242.955156:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000891e59fa.
00000400:00000010:2.0:1787866242.955157:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000003ffeab57.
00000400:00000010:2.0:1787866242.955160:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f8e9a5a4.
00000400:00000010:2.0:1787866242.955162:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000024bb7037.
00000400:00000010:2.0:1787866242.955164:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000c43a83d3.
00000400:00000010:2.0:1787866242.955165:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000f0f413bc.
00000400:00000010:2.0:1787866242.955169:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000079ea313f.
00000400:00000010:2.0:1787866242.955171:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000003a15ec44.
00000400:00000010:2.0:1787866242.955172:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000ebf511a2.
00000400:00000010:2.0:1787866242.955174:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000008428b227.
00000400:00000010:2.0:1787866242.955177:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000088e78596.
00000400:00000010:2.0:1787866242.955179:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000aafd4e56.
00000400:00000010:2.0:1787866242.955181:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000b44b6829.
00000400:00000010:2.0:1787866242.955182:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000003b33edc5.
00000400:00000010:2.0:1787866242.955186:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ad1ad778.
00000400:00000010:2.0:1787866242.955189:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000006c75b56.
00000400:00000010:2.0:1787866242.955191:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000003d983f9f.
00000400:00000010:2.0:1787866242.955196:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000008ccd18f3.
00000400:00000010:2.0:1787866242.955200:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000003cb04dca.
00000400:00000010:2.0:1787866242.955203:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000004d5d243f.
00000400:00000010:2.0:1787866242.955204:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000c6b11cde.
00000400:00000010:2.0:1787866242.955206:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e4494135.
00000400:00000010:2.0:1787866242.955209:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ff0fa99f.
00000400:00000010:2.0:1787866242.955211:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000033b0a5ba.
00000400:00000010:2.0:1787866242.955213:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000fa88b919.
00000400:00000010:2.0:1787866242.955214:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000003aea9b.
00000400:00000010:2.0:1787866242.955218:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000b355e253.
00000400:00000010:2.0:1787866242.955220:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000fb3bbee8.
00000400:00000010:2.0:1787866242.955222:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000004c7e3611.
00000400:00000010:2.0:1787866242.955223:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000384aa030.
00000400:00000010:2.0:1787866242.955228:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000dbb65273.
00000400:00000010:2.0:1787866242.955231:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000fed419e0.
00000400:00000010:2.0:1787866242.955232:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000c33ae974.
00000400:00000010:2.0:1787866242.955234:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000dc4a8d7c.
00000400:00000010:2.0:1787866242.955237:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000042dece22.
00000400:00000010:2.0:1787866242.955240:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000007620a0d3.
00000400:00000010:2.0:1787866242.955242:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000feccc3d4.
00000400:00000010:2.0:1787866242.955243:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000057596936.
00000400:00000010:2.0:1787866242.955247:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ede22331.
00000400:00000010:2.0:1787866242.955249:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000d72b201a.
00000400:00000010:2.0:1787866242.955251:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000cf1de98d.
00000400:00000010:2.0:1787866242.955253:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000d98881dd.
00000400:00000010:2.0:1787866242.955256:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000874db7d2.
00000400:00000010:2.0:1787866242.955258:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000001630e016.
00000400:00000010:2.0:1787866242.955259:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000068ee1672.
00000400:00000010:2.0:1787866242.955260:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000072c42fde.
00000400:00000010:2.0:1787866242.955264:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ed353615.
00000400:00000010:2.0:1787866242.955267:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000036c04c13.
00000400:00000010:2.0:1787866242.955269:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000f123ceb5.
00000400:00000010:2.0:1787866242.955270:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000069f2c576.
00000400:00000010:2.0:1787866242.955283:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000006e575a31.
00000400:00000010:2.0:1787866242.955287:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000002e3843d1.
00000400:00000010:2.0:1787866242.955288:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000086763905.
00000400:00000010:2.0:1787866242.955290:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000001ce22a9f.
00000400:00000010:2.0:1787866242.955294:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000091bd4b17.
00000400:00000010:2.0:1787866242.955296:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000093c4a148.
00000400:00000010:2.0:1787866242.955297:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000f2893b01.
00000400:00000010:2.0:1787866242.955299:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000009087bf54.
00000400:00000010:2.0:1787866242.955302:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f35d978d.
00000400:00000010:2.0:1787866242.955304:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000000b579dd1.
00000400:00000010:2.0:1787866242.955306:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000093d22730.
00000400:00000010:2.0:1787866242.955307:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000003dd6ae70.
00000400:00000010:2.0:1787866242.955312:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000005604167f.
00000400:00000010:2.0:1787866242.955314:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000a48aacda.
00000400:00000010:2.0:1787866242.955316:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000392966ae.
00000400:00000010:2.0:1787866242.955318:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e96b71a1.
00000400:00000010:2.0:1787866242.955322:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000049f79d76.
00000400:00000010:2.0:1787866242.955341:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000011714931.
00000400:00000010:2.0:1787866242.955343:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000007e6773b2.
00000400:00000010:2.0:1787866242.955345:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000005365e82e.
00000400:00000010:2.0:1787866242.955349:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000028b9089e.
00000400:00000010:2.0:1787866242.955352:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000098227a4a.
00000400:00000010:2.0:1787866242.955353:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000008c8a7b83.
00000400:00000010:2.0:1787866242.955355:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000087818236.
00000400:00000010:2.0:1787866242.955359:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000b6fb7914.
00000400:00000010:2.0:1787866242.955362:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000905a879b.
00000400:00000010:2.0:1787866242.955364:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000012a70cfa.
00000400:00000010:2.0:1787866242.955365:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000938cb033.
00000400:00000010:2.0:1787866242.955369:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000005e83f381.
00000400:00000010:2.0:1787866242.955371:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000009f2026b8.
00000400:00000010:2.0:1787866242.955373:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000b8459f62.
00000400:00000010:2.0:1787866242.955375:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e9590d32.
00000400:00000010:2.0:1787866242.955378:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000002aacc980.
00000400:00000010:2.0:1787866242.955387:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000006b193b9.
00000400:00000010:2.0:1787866242.955389:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002d9eae7b.
00000400:00000010:2.0:1787866242.955390:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000e4649c08.
00000400:00000010:2.0:1787866242.955394:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000000861471c.
00000400:00000010:2.0:1787866242.955396:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000003ae1401c.
00000400:00000010:2.0:1787866242.955397:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000133b8b17.
00000400:00000010:2.0:1787866242.955399:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000b2a3b6a8.
00000400:00000010:2.0:1787866242.955403:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000005083cb8a.
00000400:00000010:2.0:1787866242.955406:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000b8140383.
00000400:00000010:2.0:1787866242.955408:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000008f8e42b6.
00000400:00000010:2.0:1787866242.955409:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000003ad4337c.
00000400:00000010:2.0:1787866242.955413:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000008198bbcd.
00000400:00000010:2.0:1787866242.955415:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000069defbed.
00000400:00000010:2.0:1787866242.955417:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000bb5702e1.
00000400:00000010:2.0:1787866242.955418:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000012d03674.
00000400:00000010:2.0:1787866242.955422:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f29e49df.
00000400:00000010:2.0:1787866242.955426:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000007e8c75e4.
00000400:00000010:2.0:1787866242.955427:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000051c6a6ba.
00000400:00000010:2.0:1787866242.955429:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000009ffd5ad9.
00000400:00000010:2.0:1787866242.955432:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000063414f75.
00000400:00000010:2.0:1787866242.955435:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000080bac632.
00000400:00000010:2.0:1787866242.955437:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000a7e11a58.
00000400:00000010:2.0:1787866242.955439:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000008787625a.
00000400:00000010:2.0:1787866242.955442:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000006d892fb3.
00000400:00000010:2.0:1787866242.955455:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000b83e4947.
00000400:00000010:2.0:1787866242.955458:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000808ee949.
00000400:00000010:2.0:1787866242.955460:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000016ffbbec.
00000400:00000010:2.0:1787866242.955463:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000072a8f16f.
00000400:00000010:2.0:1787866242.955465:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000eb5843d2.
00000400:00000010:2.0:1787866242.955467:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000ca4b56be.
00000400:00000010:2.0:1787866242.955468:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000070635d03.
00000400:00000010:2.0:1787866242.955471:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000076d6cd1c.
00000400:00000010:2.0:1787866242.955473:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000a063f889.
00000400:00000010:2.0:1787866242.955479:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000a629319d.
00000400:00000010:2.0:1787866242.955481:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000a3257d5d.
00000400:00000010:2.0:1787866242.955484:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000015c7a150.
00000400:00000010:2.0:1787866242.955486:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f0782cce.
00000400:00000010:2.0:1787866242.955488:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000004d1fa072.
00000400:00000010:2.0:1787866242.955489:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000072de1828.
00000400:00000010:2.0:1787866242.955493:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000a3e62e9d.
00000400:00000010:2.0:1787866242.955496:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000ea07f75a.
00000400:00000010:2.0:1787866242.955497:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000088c059cd.
00000400:00000010:2.0:1787866242.955498:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000072029be1.
00000400:00000010:2.0:1787866242.955502:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000799598b3.
00000400:00000010:2.0:1787866242.955504:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000d72f4d76.
00000400:00000010:2.0:1787866242.955506:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000003dbeef4f.
00000400:00000010:2.0:1787866242.955507:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000a4918ca0.
00000400:00000010:2.0:1787866242.955511:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000e53a5eee.
00000400:00000010:2.0:1787866242.955513:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000b140c2da.
00000400:00000010:2.0:1787866242.955515:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000028d8caf.
00000400:00000010:2.0:1787866242.955516:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000ab15aaf2.
00000400:00000010:2.0:1787866242.955520:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000bce067fc.
00000400:00000010:2.0:1787866242.955523:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000000b07253f.
00000400:00000010:2.0:1787866242.955524:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000021f78ebd.
00000400:00000010:2.0:1787866242.955526:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000fd0ac38c.
00000400:00000010:2.0:1787866242.955530:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000052006a4e.
00000400:00000010:2.0:1787866242.955533:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000003d4a94d8.
00000400:00000010:2.0:1787866242.955535:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000e04eb8a3.
00000400:00000010:2.0:1787866242.955536:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000aaa9c056.
00000400:00000010:2.0:1787866242.955540:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000006ce923da.
00000400:00000010:2.0:1787866242.955542:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000aa19f1a3.
00000400:00000010:2.0:1787866242.955544:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000004b0df6.
00000400:00000010:2.0:1787866242.955546:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000007ae10c44.
00000400:00000010:2.0:1787866242.955549:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000067e117b8.
00000400:00000010:2.0:1787866242.955551:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000c6497b0c.
00000400:00000010:2.0:1787866242.955553:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000bbc4fead.
00000400:00000010:2.0:1787866242.955561:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000be2546b6.
00000400:00000010:2.0:1787866242.955564:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000003dca47e3.
00000400:00000010:2.0:1787866242.955567:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000009c022a70.
00000400:00000010:2.0:1787866242.955568:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000f232dcd5.
00000400:00000010:2.0:1787866242.955570:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000048d6c5d0.
00000400:00000010:2.0:1787866242.955574:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000ff320cb3.
00000400:00000010:2.0:1787866242.955590:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000df0a1b16.
00000400:00000010:2.0:1787866242.955593:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000369e4733.
00000400:00000010:2.0:1787866242.955595:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000036b3a50e.
00000400:00000010:2.0:1787866242.955598:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000010775746.
00000400:00000010:2.0:1787866242.955600:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000f9a8cb83.
00000400:00000010:2.0:1787866242.955601:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000506c05b3.
00000400:00000010:2.0:1787866242.955603:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000375896ff.
00000400:00000010:2.0:1787866242.955606:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000006fe7da78.
00000400:00000010:2.0:1787866242.955609:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000a17cc9d9.
00000400:00000010:2.0:1787866242.955611:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000c1a9aa8c.
00000400:00000010:2.0:1787866242.955612:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000012c6c579.
00000400:00000010:2.0:1787866242.955616:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f68aaa16.
00000400:00000010:2.0:1787866242.955619:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000aa4f13b0.
00000400:00000010:2.0:1787866242.955620:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000057841de5.
00000400:00000010:2.0:1787866242.955622:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000013205a57.
00000400:00000010:2.0:1787866242.955626:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 0000000066cc6be4.
00000400:00000010:2.0:1787866242.955628:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000055425607.
00000400:00000010:2.0:1787866242.955631:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000000899e8cc.
00000400:00000010:2.0:1787866242.955632:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 000000007bf2fb4c.
00000400:00000010:2.0:1787866242.955636:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000d94bb90a.
00000400:00000010:2.0:1787866242.955638:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000d7a60416.
00000400:00000010:2.0:1787866242.955639:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000ffd0d466.
00000400:00000010:2.0:1787866242.955641:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000094312786.
00000400:00000010:2.0:1787866242.955644:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000002f504b20.
00000400:00000010:2.0:1787866242.955647:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000007c81eb09.
00000400:00000010:2.0:1787866242.955648:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000008035aa29.
00000400:00000010:2.0:1787866242.955650:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000aa387aa5.
00000400:00000010:2.0:1787866242.955659:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 00000000f3b8bfe7.
00000400:00000010:2.0:1787866242.955662:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000d28068b0.
00000400:00000010:2.0:1787866242.955663:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 00000000bad506c6.
00000400:00000010:2.0:1787866242.955665:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 0000000088e7fc31.
00000400:00000010:2.0:1787866242.955669:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000007666a5df.
00000400:00000010:2.0:1787866242.955671:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 00000000e80ae833.
00000400:00000010:2.0:1787866242.955673:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000002b653e15.
00000400:00000010:2.0:1787866242.955674:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000c2ec0b30.
00000400:00000010:2.0:1787866242.955678:0:69863:0:(lib-ptl.c:786:lnet_ptl_cleanup()) kfreed 'mhash': 4112 at 000000004da4d315.
00000400:00000010:2.0:1787866242.955680:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 0000000021031acc.
00000400:00000010:2.0:1787866242.955682:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000009467422c.
00000400:00000010:2.0:1787866242.955683:0:69863:0:(lib-ptl.c:851:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals[i]': 84 at 00000000ae718185.
00000400:00000010:2.0:1787866242.955685:0:69863:0:(lib-ptl.c:854:lnet_portals_destroy()) kfreed 'the_lnet.ln_portals': 512 at 00000000149ca4aa.
00000400:00000010:2.0:1787866242.955690:0:69863:0:(api-ni.c:1205:lnet_res_container_cleanup()) kfreed 'rec->rec_lh_hash': 65536 at 000000001fd70cf7.
00000400:00000010:2.0:1787866242.955705:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 64 at 000000009ad4c87e.
00000400:00000010:2.0:1787866242.955708:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000030a803c4.
00000400:00000010:2.0:1787866242.955710:0:69863:0:(api-ni.c:1205:lnet_res_container_cleanup()) kfreed 'rec->rec_lh_hash': 65536 at 00000000479850c5.
00000400:00000010:2.0:1787866242.955714:0:69863:0:(lib-msg.c:1308:lnet_msg_container_cleanup()) kfreed 'container->msc_finalizers': 32 at 000000002b7fb679.
00000400:00000010:2.0:1787866242.955716:0:69863:0:(lib-msg.c:1314:lnet_msg_container_cleanup()) kfreed 'container->msc_resenders': 32 at 0000000066cecd94.
00000400:00000010:2.0:1787866242.955718:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 00000000c0dd528f.
00000400:00000010:2.0:1787866242.955720:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000035a1ff94.
00000400:00000010:2.0:1787866242.955725:0:69863:0:(peer.c:85:lnet_peer_tables_destroy()) kfreed 'hash': 8192 at 00000000a721e894.
00000400:00000010:2.0:1787866242.955735:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 000000004a143be2.
00000400:00000010:2.0:1787866242.955737:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 0000000038e32123.
00000400:00000010:2.0:1787866242.955740:0:69863:0:(lib-cpt.c:1284:cfs_percpt_free()) kfreed 'arr->va_ptrs[i]': 128 at 00000000e68bbf54.
00000400:00000010:2.0:1787866242.955742:0:69863:0:(lib-cpt.c:1288:cfs_percpt_free()) kfreed 'arr': 24 at 000000007a722ddd.
00000400:00000010:2.0:1787866242.955746:0:69863:0:(api-ni.c:700:lnet_destroy_remote_nets_table()) kfreed 'the_lnet.ln_remote_nets_hash': 1024 at 00000000550bd6db.
00000001:00000200:2.0:1787866242.955750:0:69863:0:(udsp.c:1126:lnet_udsp_destroy()) Destroying UDSPs in the system
00000001:02000400:3.0:1787866242.982551:0:69864:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet configure
00000400:00000010:0.0:1787866243.011190:0:69865:0:(api-ni.c:676:lnet_create_remote_nets_table()) alloc '(hash)': 1024 at 0000000060e11b69.
00000400:00000010:0.0:1787866243.011204:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000cda00ea6.
00000400:00000010:0.0:1787866243.011210:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000850806b0.
00000400:00000010:0.0:1787866243.011213:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000097a9d4b8.
00000400:00000010:0.0:1787866243.011215:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000e3c77366.
00000400:00000010:0.0:1787866243.011222:0:69865:0:(peer.c:109:lnet_peer_tables_create()) alloc '(hash)': 8192 at 0000000087eabc9c.
00000400:00000010:0.0:1787866243.011227:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ae4f08e8.
00000400:00000010:0.0:1787866243.011228:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 000000000c666557.
00000400:00000010:0.0:1787866243.011232:0:69865:0:(lib-msg.c:1338:lnet_msg_container_setup()) alloc '(container->msc_finalizers)': 32 at 000000008de623f2.
00000400:00000010:0.0:1787866243.011236:0:69865:0:(lib-msg.c:1348:lnet_msg_container_setup()) alloc '(container->msc_resenders)': 32 at 000000001a88a19b.
00000400:00000010:0.0:1787866243.011325:0:69865:0:(api-ni.c:1227:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 0000000006a275f4.
00000400:00000010:0.0:1787866243.011345:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000c3a12aeb.
00000400:00000010:0.0:1787866243.011348:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000305c9440.
00000400:00000010:0.0:1787866243.011419:0:69865:0:(api-ni.c:1227:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 000000001d908fd9.
00000400:00000010:0.0:1787866243.011439:0:69865:0:(lib-ptl.c:864:lnet_portals_create()) alloc '(the_lnet.ln_portals)': 512 at 000000007365fc70.
00000400:00000010:0.0:1787866243.011443:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000009c97d065.
00000400:00000010:0.0:1787866243.011447:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000056ff403c.
00000400:00000010:0.0:1787866243.011450:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006b090781.
00000400:00000010:0.0:1787866243.011456:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000bba29d83.
00000400:00000010:0.0:1787866243.011458:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000b3f2d3e4.
00000400:00000010:0.0:1787866243.011461:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000308f8eea.
00000400:00000010:0.0:1787866243.011462:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000052e194fb.
00000400:00000010:0.0:1787866243.011467:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008fe39939.
00000400:00000010:0.0:1787866243.011469:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000730e1b6b.
00000400:00000010:0.0:1787866243.011477:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000f4d39c89.
00000400:00000010:0.0:1787866243.011479:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006e78d75b.
00000400:00000010:0.0:1787866243.011483:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008d8cdc37.
00000400:00000010:0.0:1787866243.011486:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000008343de91.
00000400:00000010:0.0:1787866243.011487:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000007ee04bf9.
00000400:00000010:0.0:1787866243.011489:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000054348412.
00000400:00000010:0.0:1787866243.011496:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007434b255.
00000400:00000010:0.0:1787866243.011499:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000002e7fa5f.
00000400:00000010:0.0:1787866243.011501:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e935f346.
00000400:00000010:0.0:1787866243.011502:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fd10822b.
00000400:00000010:0.0:1787866243.011508:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000cb480af2.
00000400:00000010:0.0:1787866243.011510:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000015cb3270.
00000400:00000010:0.0:1787866243.011512:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000b788b344.
00000400:00000010:0.0:1787866243.011513:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000751743ef.
00000400:00000010:0.0:1787866243.011517:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a27db30f.
00000400:00000010:0.0:1787866243.011520:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e209edf4.
00000400:00000010:0.0:1787866243.011521:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ee4f9863.
00000400:00000010:0.0:1787866243.011523:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000271b1620.
00000400:00000010:0.0:1787866243.011527:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ce65c75a.
00000400:00000010:0.0:1787866243.011529:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000036dc3599.
00000400:00000010:0.0:1787866243.011531:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ffe2a7bf.
00000400:00000010:0.0:1787866243.011533:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000145c4263.
00000400:00000010:0.0:1787866243.011538:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a01446bf.
00000400:00000010:0.0:1787866243.011540:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000646f8156.
00000400:00000010:0.0:1787866243.011541:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000b67272b9.
00000400:00000010:0.0:1787866243.011543:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000bfcbe30d.
00000400:00000010:0.0:1787866243.011548:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000061615e9.
00000400:00000010:0.0:1787866243.011550:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000009cafad7d.
00000400:00000010:0.0:1787866243.011551:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000cc589812.
00000400:00000010:0.0:1787866243.011553:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e73558bd.
00000400:00000010:0.0:1787866243.011557:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002bcb5613.
00000400:00000010:0.0:1787866243.011559:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000069cf72d3.
00000400:00000010:0.0:1787866243.011566:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000040f3806a.
00000400:00000010:0.0:1787866243.011568:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002dc85572.
00000400:00000010:0.0:1787866243.011573:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ec41b373.
00000400:00000010:0.0:1787866243.011576:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000005fcca53.
00000400:00000010:0.0:1787866243.012007:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001d73d140.
00000400:00000010:0.0:1787866243.012012:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000033e6fbf0.
00000400:00000010:0.0:1787866243.012019:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b355e253.
00000400:00000010:0.0:1787866243.012022:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000dc723810.
00000400:00000010:0.0:1787866243.012023:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000d407977b.
00000400:00000010:0.0:1787866243.012025:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000010b9e342.
00000400:00000010:0.0:1787866243.012029:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ff0fa99f.
00000400:00000010:0.0:1787866243.012032:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000908dc6be.
00000400:00000010:0.0:1787866243.012033:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000004483ae1d.
00000400:00000010:0.0:1787866243.012035:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000014122af9.
00000400:00000010:0.0:1787866243.012039:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003cb04dca.
00000400:00000010:0.0:1787866243.012042:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000001b5a5e34.
00000400:00000010:0.0:1787866243.012044:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000f863196e.
00000400:00000010:0.0:1787866243.012045:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e7263520.
00000400:00000010:0.0:1787866243.012050:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ad1ad778.
00000400:00000010:0.0:1787866243.012053:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000efa2d39e.
00000400:00000010:0.0:1787866243.012054:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000636ac174.
00000400:00000010:0.0:1787866243.012056:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009c0bc3d7.
00000400:00000010:0.0:1787866243.012062:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000088e78596.
00000400:00000010:0.0:1787866243.012064:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000009d65ff37.
00000400:00000010:0.0:1787866243.012066:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000032c8608f.
00000400:00000010:0.0:1787866243.012068:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000aee49ec7.
00000400:00000010:0.0:1787866243.012072:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000079ea313f.
00000400:00000010:0.0:1787866243.012075:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000635557bb.
00000400:00000010:0.0:1787866243.012077:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e0d16f3a.
00000400:00000010:0.0:1787866243.012079:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007b055901.
00000400:00000010:0.0:1787866243.012092:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f8e9a5a4.
00000400:00000010:0.0:1787866243.012096:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000005fe8e470.
00000400:00000010:0.0:1787866243.012104:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000006f4b5731.
00000400:00000010:0.0:1787866243.012105:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e5ee0447.
00000400:00000010:0.0:1787866243.012111:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e4747cb8.
00000400:00000010:0.0:1787866243.012113:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000000e7381ec.
00000400:00000010:0.0:1787866243.012115:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000567a3e14.
00000400:00000010:0.0:1787866243.012116:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000050fa2c2d.
00000400:00000010:0.0:1787866243.012122:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000000ee9133.
00000400:00000010:0.0:1787866243.012124:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000052532570.
00000400:00000010:0.0:1787866243.012126:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000085f49266.
00000400:00000010:0.0:1787866243.012128:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fa1cc355.
00000400:00000010:0.0:1787866243.012133:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003506a16b.
00000400:00000010:0.0:1787866243.012136:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000c941064f.
00000400:00000010:0.0:1787866243.012137:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000027d21b25.
00000400:00000010:0.0:1787866243.012139:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000561a9f90.
00000400:00000010:0.0:1787866243.012144:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000000b3665f6.
00000400:00000010:0.0:1787866243.012146:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000048e3f1da.
00000400:00000010:0.0:1787866243.012148:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000b6d5ead9.
00000400:00000010:0.0:1787866243.012150:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002b9ed566.
00000400:00000010:0.0:1787866243.012154:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000eff1234f.
00000400:00000010:0.0:1787866243.012157:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000006b2f05fd.
00000400:00000010:0.0:1787866243.012158:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000095094022.
00000400:00000010:0.0:1787866243.012160:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007fc9ad76.
00000400:00000010:0.0:1787866243.012167:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005604167f.
00000400:00000010:0.0:1787866243.012170:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000b1e206fe.
00000400:00000010:0.0:1787866243.012171:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e3d91ee9.
00000400:00000010:0.0:1787866243.012173:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001a85149c.
00000400:00000010:0.0:1787866243.012179:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000028b9089e.
00000400:00000010:0.0:1787866243.012182:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000004195d26b.
00000400:00000010:0.0:1787866243.012183:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000007f052704.
00000400:00000010:0.0:1787866243.012185:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ebfcfd72.
00000400:00000010:0.0:1787866243.012190:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000049f79d76.
00000400:00000010:0.0:1787866243.012193:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000070c05b7b.
00000400:00000010:0.0:1787866243.012200:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000008ba195b9.
00000400:00000010:0.0:1787866243.012202:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000f65dfed.
00000400:00000010:0.0:1787866243.012219:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000066b6d98f.
00000400:00000010:0.0:1787866243.012222:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000000a09bcd6.
00000400:00000010:0.0:1787866243.012224:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000023a1c5d.
00000400:00000010:0.0:1787866243.012226:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000027dd214c.
00000400:00000010:0.0:1787866243.012231:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f783d8b3.
00000400:00000010:0.0:1787866243.012233:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a97602a6.
00000400:00000010:0.0:1787866243.012234:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001122b902.
00000400:00000010:0.0:1787866243.012236:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009a70286f.
00000400:00000010:0.0:1787866243.012241:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a721e894.
00000400:00000010:0.0:1787866243.012244:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000032f290df.
00000400:00000010:0.0:1787866243.012246:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000000e720356.
00000400:00000010:0.0:1787866243.012247:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a721cdeb.
00000400:00000010:0.0:1787866243.012252:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b1b59a22.
00000400:00000010:0.0:1787866243.012255:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000002a037581.
00000400:00000010:0.0:1787866243.012257:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000279b7e94.
00000400:00000010:0.0:1787866243.012259:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008c3cd629.
00000400:00000010:0.0:1787866243.012270:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006ce923da.
00000400:00000010:0.0:1787866243.012275:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ae831708.
00000400:00000010:0.0:1787866243.012277:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000278c11a7.
00000400:00000010:0.0:1787866243.012279:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009d88af48.
00000400:00000010:0.0:1787866243.012284:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000067e117b8.
00000400:00000010:0.0:1787866243.012286:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000067a22d7a.
00000400:00000010:0.0:1787866243.012288:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000bd4784bd.
00000400:00000010:0.0:1787866243.012290:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008cf6d149.
00000400:00000010:0.0:1787866243.012295:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003dca47e3.
00000400:00000010:0.0:1787866243.012297:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000aa3d0402.
00000400:00000010:0.0:1787866243.012299:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000009c201ede.
00000400:00000010:0.0:1787866243.012301:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c898f900.
00000400:00000010:0.0:1787866243.012306:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000052006a4e.
00000400:00000010:0.0:1787866243.012308:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000004ad99b0a.
00000400:00000010:0.0:1787866243.012315:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000067dc65ea.
00000400:00000010:0.0:1787866243.012317:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000063a82cf7.
00000400:00000010:0.0:1787866243.012329:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a3e62e9d.
00000400:00000010:0.0:1787866243.012332:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a2379bf3.
00000400:00000010:0.0:1787866243.012334:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000054c33df7.
00000400:00000010:0.0:1787866243.012336:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003a6988f4.
00000400:00000010:0.0:1787866243.012340:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000799598b3.
00000400:00000010:0.0:1787866243.012343:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000003e34501a.
00000400:00000010:0.0:1787866243.012345:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000cbe89673.
00000400:00000010:0.0:1787866243.012346:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000036d26412.
00000400:00000010:0.0:1787866243.012350:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e53a5eee.
00000400:00000010:0.0:1787866243.012353:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000095b4b2cc.
00000400:00000010:0.0:1787866243.012355:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001b9612e5.
00000400:00000010:0.0:1787866243.012356:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a94999eb.
00000400:00000010:0.0:1787866243.012361:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000bce067fc.
00000400:00000010:0.0:1787866243.012363:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000000e09f9ba.
00000400:00000010:0.0:1787866243.012365:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000842e7e6b.
00000400:00000010:0.0:1787866243.012367:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000014dcdf80.
00000400:00000010:0.0:1787866243.012378:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000072a8f16f.
00000400:00000010:0.0:1787866243.012382:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000047c78a34.
00000400:00000010:0.0:1787866243.012383:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000007df7e97e.
00000400:00000010:0.0:1787866243.012385:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000048ba996b.
00000400:00000010:0.0:1787866243.012390:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000076d6cd1c.
00000400:00000010:0.0:1787866243.012393:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000a7106051.
00000400:00000010:0.0:1787866243.012395:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002491ef89.
00000400:00000010:0.0:1787866243.012397:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ce239058.
00000400:00000010:0.0:1787866243.012401:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000015c7a150.
00000400:00000010:0.0:1787866243.012405:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000763c5aaf.
00000400:00000010:0.0:1787866243.012406:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000005e773e93.
00000400:00000010:0.0:1787866243.012408:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c14734c3.
00000400:00000010:0.0:1787866243.012412:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006d892fb3.
00000400:00000010:0.0:1787866243.012415:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000b36f0083.
00000400:00000010:0.0:1787866243.012422:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000efa46b67.
00000400:00000010:0.0:1787866243.012424:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004b553335.
00000400:00000010:0.0:1787866243.012435:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000063414f75.
00000400:00000010:0.0:1787866243.012444:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000006fd1d480.
00000400:00000010:0.0:1787866243.012447:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000003bac1389.
00000400:00000010:0.0:1787866243.012449:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d014d1b7.
00000400:00000010:0.0:1787866243.012453:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005083cb8a.
00000400:00000010:0.0:1787866243.012456:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000df499d21.
00000400:00000010:0.0:1787866243.012458:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000009483fef6.
00000400:00000010:0.0:1787866243.012460:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009ad4c87e.
00000400:00000010:0.0:1787866243.012464:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008198bbcd.
00000400:00000010:0.0:1787866243.012467:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000002cad1fac.
00000400:00000010:0.0:1787866243.012469:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000009e922710.
00000400:00000010:0.0:1787866243.012470:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c6497b0c.
00000400:00000010:0.0:1787866243.012475:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f29e49df.
00000400:00000010:0.0:1787866243.012478:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000009d3d12ef.
00000400:00000010:0.0:1787866243.012480:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000043a9ed68.
00000400:00000010:0.0:1787866243.012482:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000aa19f1a3.
00000400:00000010:0.0:1787866243.012493:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b6fb7914.
00000400:00000010:0.0:1787866243.012497:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000001a10bec5.
00000400:00000010:0.0:1787866243.012499:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000b8820be6.
00000400:00000010:0.0:1787866243.012501:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003d4a94d8.
00000400:00000010:0.0:1787866243.012506:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005e83f381.
00000400:00000010:0.0:1787866243.012508:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000dd5ccefc.
00000400:00000010:0.0:1787866243.012510:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e5f847fd.
00000400:00000010:0.0:1787866243.012512:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000b07253f.
00000400:00000010:0.0:1787866243.012517:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002aacc980.
00000400:00000010:0.0:1787866243.012519:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000007ff975b4.
00000400:00000010:0.0:1787866243.012521:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000e32d58ee.
00000400:00000010:0.0:1787866243.012522:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b140c2da.
00000400:00000010:0.0:1787866243.012527:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000000861471c.
00000400:00000010:0.0:1787866243.012529:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000038becf08.
00000400:00000010:0.0:1787866243.012539:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000004742348e.
00000400:00000010:0.0:1787866243.012541:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d72f4d76.
00000400:00000010:0.0:1787866243.012553:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000dbb65273.
00000400:00000010:0.0:1787866243.012556:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000ed3c6b54.
00000400:00000010:0.0:1787866243.012558:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000001885c878.
00000400:00000010:0.0:1787866243.012560:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ea07f75a.
00000400:00000010:0.0:1787866243.012564:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000874db7d2.
00000400:00000010:0.0:1787866243.012567:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000b5d99914.
00000400:00000010:0.0:1787866243.012569:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000071485387.
00000400:00000010:0.0:1787866243.012570:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f0782cce.
00000400:00000010:0.0:1787866243.012575:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ede22331.
00000400:00000010:0.0:1787866243.013134:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000005fd38689.
00000400:00000010:0.0:1787866243.013141:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000066af8612.
00000400:00000010:0.0:1787866243.013144:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a063f889.
00000400:00000010:0.0:1787866243.013153:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000042dece22.
00000400:00000010:0.0:1787866243.013156:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000f05f6bb6.
00000400:00000010:0.0:1787866243.013158:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000007a722ddd.
00000400:00000010:0.0:1787866243.013160:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000eb5843d2.
00000400:00000010:0.0:1787866243.013173:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000af0841f6.
00000400:00000010:0.0:1787866243.013178:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000b28f8b6b.
00000400:00000010:0.0:1787866243.013180:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000038e32123.
00000400:00000010:0.0:1787866243.013182:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b83e4947.
00000400:00000010:0.0:1787866243.013188:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001fc9d250.
00000400:00000010:0.0:1787866243.013190:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000000447a0ee.
00000400:00000010:0.0:1787866243.013192:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000035a1ff94.
00000400:00000010:0.0:1787866243.013193:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000080bac632.
00000400:00000010:0.0:1787866243.013198:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000066fd5643.
00000400:00000010:0.0:1787866243.013200:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000001efe26aa.
00000400:00000010:0.0:1787866243.013202:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000066cecd94.
00000400:00000010:0.0:1787866243.013204:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007e8c75e4.
00000400:00000010:0.0:1787866243.013209:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000372d9216.
00000400:00000010:0.0:1787866243.013212:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000007c8686b3.
00000400:00000010:0.0:1787866243.013219:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000030a803c4.
00000400:00000010:0.0:1787866243.013222:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000069defbed.
00000400:00000010:0.0:1787866243.013233:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e0ccdfeb.
00000400:00000010:0.0:1787866243.013236:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000009eae5317.
00000400:00000010:0.0:1787866243.013238:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000002b653e15.
00000400:00000010:0.0:1787866243.013240:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b8140383.
00000400:00000010:0.0:1787866243.013244:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000068812113.
00000400:00000010:0.0:1787866243.013247:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 0000000024e95851.
00000400:00000010:0.0:1787866243.013248:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000bad506c6.
00000400:00000010:0.0:1787866243.013250:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003ae1401c.
00000400:00000010:0.0:1787866243.013256:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ddcf63b1.
00000400:00000010:0.0:1787866243.013258:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 000000004518b1ee.
00000400:00000010:0.0:1787866243.013260:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000008035aa29.
00000400:00000010:0.0:1787866243.013261:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000006b193b9.
00000400:00000010:0.0:1787866243.013267:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008214abc9.
00000400:00000010:0.0:1787866243.013269:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000e5c780ca.
00000400:00000010:0.0:1787866243.013270:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ffd0d466.
00000400:00000010:0.0:1787866243.013272:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009f2026b8.
00000400:00000010:0.0:1787866243.013283:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000017c84c1a.
00000400:00000010:0.0:1787866243.013288:0:69865:0:(lib-ptl.c:871:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 84 at 00000000515506ff.
00000400:00000010:0.0:1787866243.013290:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000000899e8cc.
00000400:00000010:0.0:1787866243.013291:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000905a879b.
00000400:00000010:0.0:1787866243.013296:0:69865:0:(lib-ptl.c:815:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e90bc9e3.
00000400:00000010:0.0:1787866243.013299:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 0000000057841de5.
00000400:00000010:0.0:1787866243.013301:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000098227a4a.
00000400:00000010:0.0:1787866243.013306:0:69865:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 000000009c27b214.
00000400:00000010:0.0:1787866243.013311:0:69865:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 552 at 000000007f74b219.
00000400:00000010:0.0:1787866243.013313:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000c1a9aa8c.
00000400:00000010:0.0:1787866243.013316:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000011714931.
00000400:00000010:0.0:1787866243.013317:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000506c05b3.
00000400:00000010:0.0:1787866243.013319:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a48aacda.
00000400:00000010:0.0:1787866243.013339:0:69865:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 41 at 000000000b579dd1.
00000400:00000010:0.0:1787866243.013346:0:69865:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000004b0661b6.
00000400:00000010:0.0:1787866243.013372:0:69865:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004e06e925.
00000400:00000200:0.0:1787866243.013388:0:69865:0:(lib-ptl.c:918:LNetSetLazyPortal()) Setting portal 0 lazy
00000400:00000010:0.0:1787866243.013392:0:69865:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000e571f8f2.
00000400:00000010:0.0:1787866243.013395:0:69865:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000076084a9f.
00000400:00000010:0.0:1787866243.013398:0:69865:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006e6648de.
00000400:00000200:0.0:1787866243.013400:0:69865:0:(api-ni.c:2383:lnet_push_target_post()) posted push target 00000000e571f8f2
00000400:00000200:0.0:1787866243.013403:0:69865:0:(api-ni.c:2340:lnet_push_target_resize()) nbytes 272 success
00000400:00000010:0.0:1787866243.013405:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000f232dcd5.
00000400:00000010:0.0:1787866243.013408:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000093c4a148.
00000400:00000010:0.0:1787866243.013410:0:69865:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000bbc4fead.
00000400:00000010:0.0:1787866243.013412:0:69865:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002e3843d1.
00000400:00000200:0.0:1787866243.014112:0:69865:0:(peer.c:4457:lnet_peer_discovery_start()) discovery start: 0
00000400:00000200:1.0:1787866243.017328:0:69867:0:(peer.c:4283:lnet_peer_discovery()) started
00000400:00000010:1.0:1787866243.029041:0:69868:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000006452280a.
00000400:00000010:1.0:1787866243.029076:0:69868:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000006452280a.
00000400:00000010:1.0:1787866243.030199:0:69868:0:(api-ni.c:7409:lnet_route_show_start()) alloc '(rlist)': 16 at 00000000e97aa57b.
00000400:00000010:1.0:1787866243.030226:0:69868:0:(api-ni.c:7309:lnet_route_show_done()) kfreed 'rlist': 16 at 00000000e97aa57b.
00000400:00000010:1.0:1787866243.030330:0:69868:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 236 at 000000002b1d620e.
00000400:00000010:1.0:1787866243.030339:0:69868:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 236 at 000000002b1d620e.
00000400:00000010:1.0:1787866243.030493:0:69868:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 00000000e97aa57b.
00000400:00000010:1.0:1787866243.030507:0:69868:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 00000000e97aa57b.
00000400:00000010:1.0:1787866243.030703:0:69868:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 12 at 00000000e97aa57b.
00000400:00000010:1.0:1787866243.030709:0:69868:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 12 at 00000000e97aa57b.
00000400:00000010:1.0:1787866243.032425:0:69868:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 32 at 0000000009de2d43.
00000001:00000200:1.0:1787866243.032439:0:69868:0:(udsp.c:863:lnet_udsp_get_policy()) Get UDSP at idx = 0
00000001:00000200:1.0:1787866243.032441:0:69868:0:(udsp.c:878:lnet_udsp_get_policy()) Found UDSP (00000000fad3d7f9)
00000400:00000010:1.0:1787866243.032444:0:69868:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 32 at 0000000009de2d43.
00000001:02000400:1.0:1787866243.052390:0:69869:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp --if ens2
00000400:00000010:3.0:1787866243.072363:0:69870:0:(api-ni.c:6877:lnet_net_cmd()) alloc '(conf)': 808 at 0000000037fd4bf1.
00000400:00000010:3.0:1787866243.072389:0:69870:0:(api-ni.c:6555:lnet_genl_parse_local_ni()) alloc '(tun)': 172 at 00000000e9935768.
00000400:00000010:3.0:1787866243.072403:0:69870:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 00000000e057cf08.
00000400:00000010:3.0:1787866243.072409:0:69870:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 552 at 00000000d4c2b553.
00000400:00000010:3.0:1787866243.072414:0:69870:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000da0f06e2.
00000400:00000010:3.0:1787866243.072421:0:69870:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000cc6f53be.
00000400:00000010:3.0:1787866243.072423:0:69870:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000ac2eadf2.
00000400:00000010:3.0:1787866243.072425:0:69870:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000be2f6e10.
00000400:00000010:3.0:1787866243.072430:0:69870:0:(config.c:397:lnet_ni_add_interface()) alloc '(ni->ni_interface)': 5 at 000000008618ecd0.
00000400:00000010:3.0:1787866243.072442:0:69870:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000450f8c61.
00000400:00000010:3.0:1787866243.072445:0:69870:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 000000002957e4db.
00000800:00000200:0.0:1787866243.074380:0:69870:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=lo ifindex 1 state 0 ns 00000000198b8817
00000800:00000200:0.0:1787866243.074391:0:69870:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=lo ifindex 1 state 0 ns 00000000198b8817
00000800:00000200:0.0:1787866243.074394:0:69870:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=ens2 ifindex 2 state 6 ns 00000000198b8817
00000800:00000200:0.0:1787866243.074396:0:69870:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=ens2 ifindex 2 state 6 ns 00000000198b8817
00000800:00000010:0.0:1787866243.074405:0:69870:0:(socklnd.c:2621:ksocknal_startup()) alloc '(net)': 208 at 00000000ccda2fb8.
00000800:00000200:0.0:1787866243.074422:0:69870:0:(socklnd_modparams.c:295:ksocklnd_lookup_conns_per_peer()) intf ens2 speed -1
00000400:00000200:1.0:1787866243.075412:0:69870:0:(config.c:1657:lnet_get_link_status_locked()) device idx 1 get_link 2
00000400:00000200:1.0:1787866243.075495:0:69870:0:(acceptor.c:209:lnet_acceptor_add_socket()) Added socket on ens2:988 (IP: 192.168.203.58)
00000400:02000000:1.0:1787866243.075514:0:69870:0:(api-ni.c:2823:lnet_startup_lndni()) Added LNI 192.168.203.58@tcp [8/256/0/180]
00000400:00000010:0.0:1787866243.080530:0:69870:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 57 at 00000000d72b201a.
00000400:00000010:0.0:1787866243.080549:0:69870:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000dab09a7d.
00000400:00000010:0.0:1787866243.080558:0:69870:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006f59bbec.
00000400:00000010:2.0:1787866243.084259:0:69870:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000004b0661b6.
00000400:00000200:2.0:1787866243.084274:0:69870:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004e06e925
00000400:00000010:2.0:1787866243.084277:0:69870:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000004e06e925.
00000400:00000010:2.0:1787866243.084284:0:69870:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 41 at 000000000b579dd1.
00000400:00000010:2.0:1787866243.084350:0:69870:0:(api-ni.c:6852:lnet_genl_parse_local_ni()) kfreed 'tun': 172 at 00000000e9935768.
00000400:00000010:2.0:1787866243.084359:0:69870:0:(api-ni.c:6994:lnet_net_cmd()) kfreed 'conf': 808 at 0000000037fd4bf1.
00000400:00000010:1.0:1787866243.155028:0:69884:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000006452280a.
00000400:00000010:1.0:1787866243.155115:0:69884:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000006452280a.
00000001:02000400:0.0:1787866244.126953:0:69990:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net add --net tcp1 --if ens2
00000400:00000010:3.0:1787866244.170769:0:69991:0:(api-ni.c:6877:lnet_net_cmd()) alloc '(conf)': 808 at 0000000092bb6b37.
00000400:00000010:3.0:1787866244.170800:0:69991:0:(api-ni.c:6555:lnet_genl_parse_local_ni()) alloc '(tun)': 172 at 0000000080a84c68.
00000400:00000010:3.0:1787866244.170813:0:69991:0:(config.c:343:lnet_net_alloc()) alloc '(net)': 176 at 000000002a2d1d50.
00000400:00000010:3.0:1787866244.170819:0:69991:0:(config.c:424:lnet_ni_alloc_common()) alloc '(ni)': 552 at 00000000c4ff8df5.
00000400:00000010:3.0:1787866244.170824:0:69991:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 000000006d94f0cb.
00000400:00000010:3.0:1787866244.170857:0:69991:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001d65a08f.
00000400:00000010:3.0:1787866244.170860:0:69991:0:(lib-cpt.c:1311:cfs_percpt_alloc()) alloc '(arr)': 24 at 00000000a1602ecf.
00000400:00000010:3.0:1787866244.170863:0:69991:0:(lib-cpt.c:1321:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e294626b.
00000400:00000010:3.0:1787866244.170867:0:69991:0:(config.c:397:lnet_ni_add_interface()) alloc '(ni->ni_interface)': 5 at 00000000b6a3e0fe.
00000800:00000010:3.0:1787866244.170880:0:69991:0:(socklnd.c:2621:ksocknal_startup()) alloc '(net)': 208 at 000000001b022c72.
00000800:00000200:3.0:1787866244.170896:0:69991:0:(socklnd_modparams.c:295:ksocklnd_lookup_conns_per_peer()) intf ens2 speed -1
00000400:00000200:3.0:1787866244.170905:0:69991:0:(config.c:1657:lnet_get_link_status_locked()) device idx 1 get_link 2
00000400:00000200:3.0:1787866244.171128:0:69991:0:(acceptor.c:209:lnet_acceptor_add_socket()) Added socket on ens2:988 (IP: 192.168.203.58)
00000400:02000000:3.0:1787866244.171205:0:69991:0:(api-ni.c:2823:lnet_startup_lndni()) Added LNI 192.168.203.58@tcp1 [8/256/0/180]
00000400:00000010:3.0:1787866244.177371:0:69991:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 73 at 000000006d63059b.
00000400:00000010:3.0:1787866244.177390:0:69991:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000094649d34.
00000400:00000010:3.0:1787866244.177425:0:69991:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000926d3b6b.
00000400:00000010:3.0:1787866244.177450:0:69991:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000dab09a7d.
00000400:00000200:3.0:1787866244.177455:0:69991:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006f59bbec
00000400:00000010:3.0:1787866244.177457:0:69991:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000006f59bbec.
00000400:00000010:3.0:1787866244.177471:0:69991:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 57 at 00000000d72b201a.
00000400:00000010:3.0:1787866244.177554:0:69991:0:(api-ni.c:6852:lnet_genl_parse_local_ni()) kfreed 'tun': 172 at 0000000080a84c68.
00000400:00000010:3.0:1787866244.177563:0:69991:0:(api-ni.c:6994:lnet_net_cmd()) kfreed 'conf': 808 at 0000000092bb6b37.
00000400:00000010:2.0:1787866244.281458:0:69997:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 00000000eef0ada5.
00000400:00000010:2.0:1787866244.281544:0:69997:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 00000000eef0ada5.
00000400:00000010:1.0:1787866245.299967:0:70115:0:(api-ni.c:5995:lnet_net_show_start()) alloc '(nlist)': 8 at 000000006452280a.
00000400:00000010:1.0:1787866245.300098:0:70115:0:(api-ni.c:5973:lnet_net_show_done()) kfreed 'nlist': 8 at 000000006452280a.
00000001:02000400:2.0:1787866245.328566:0:70117:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl discover 192.168.203.58@tcp
00000400:00000010:0.0:1787866245.370317:0:70118:0:(peer.c:230:lnet_peer_alloc()) alloc '(lp)': 360 at 00000000ac559109.
00000400:00000200:0.0:1787866245.370337:0:70118:0:(peer.c:266:lnet_peer_alloc()) 00000000ac559109 nid 192.168.203.58@tcp
00000400:00000010:0.0:1787866245.370341:0:70118:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 0000000077d69ee9.
00000400:00000200:0.0:1787866245.370344:0:70118:0:(peer.c:203:lnet_peer_net_alloc()) 0000000077d69ee9 net tcp
00000400:00000010:0.0:1787866245.370347:0:70118:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 00000000c5358a49.
00000400:00000200:0.0:1787866245.370351:0:70118:0:(peer.c:184:lnet_peer_ni_alloc()) 00000000c5358a49 nid 192.168.203.58@tcp
00000400:00000200:0.0:1787866245.370358:0:70118:0:(peer.c:1804:lnet_peer_attach_peer_ni()) peer 192.168.203.58@tcp NID 192.168.203.58@tcp flags 0x0
00000400:00000200:0.0:1787866245.370362:0:70118:0:(peer.c:2183:lnet_peer_ni_traffic_add()) peer 192.168.203.58@tcp: 0
00000400:00000200:0.0:1787866245.370766:0:70118:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.203.58@tcp: 0
00000400:00000200:0.0:1787866245.370769:0:70118:0:(peer.c:2935:lnet_discover_peer_locked()) Discovery attempt # 1
00000400:00000200:2.0:1787866245.370803:0:69867:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0
00000400:00000200:2.0:1787866245.370810:0:69867:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000ac559109) state 0x40
00000400:00000010:2.0:1787866245.370819:0:69867:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000033cb82bd.
00000400:00000010:2.0:1787866245.371039:0:69867:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000c3a7df46.
00000400:00000010:0.0:1787866245.377240:0:69867:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000f6fcf9ac.
00000400:00000200:0.0:1787866245.377275:0:69867:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.203.58@tcp
00000400:00000200:0.0:1787866245.377297:0:69867:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR:  192.168.203.58@tcp local destination
00000400:00000200:0.0:1787866245.377306:0:69867:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.58@tcp
00000400:00000200:0.0:1787866245.377309:0:69867:0:(lib-move.c:1874:lnet_set_non_mr_pref_nid()) Setting preferred local NID 192.168.203.58@tcp on NMR peer 192.168.203.58@tcp
00000400:00000200:0.0:1787866245.377313:0:69867:0:(peer.c:1171:lnet_peer_ni_set_non_mr_pref_nid()) peer 192.168.203.58@tcp nid 192.168.203.58@tcp: 0
00000400:00000200:0.0:1787866245.377318:0:69867:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [32:32:256:4294967295] 192.168.203.58@tcp LPNI seq info [1:1:8:4294967295]
00000400:00000200:0.0:1787866245.377325:0:69867:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:0.0:1787866245.377332:0:69867:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:<?>) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : GET try# 0
00000800:00000200:0.0:1787866245.377338:0:69867:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.203.58@tcp
00000800:00000010:0.0:1787866245.377345:0:69867:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000050d5d605.
00000800:00000010:0.0:1787866245.377358:0:69867:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 0000000065f3099b.
00000800:00000010:0.0:1787866245.377363:0:69867:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 0000000058f5f76c.
00000800:00000200:0.0:1787866245.377382:0:69867:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000065f3099b] -> 12345-192.168.203.58@tcp (2)
00000800:00000200:0.1:1787866245.377387:0:69867:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000065f3099b] -> 12345-192.168.203.58@tcp (2)
00000400:00000200:0.0:1787866245.378323:0:69867:0:(peer.c:4022:lnet_peer_send_ping()) peer 192.168.203.58@tcp
00000400:00000200:0.0:1787866245.378330:0:69867:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000ac559109) state 0x240 rc 0
00000800:00000010:1.0:1787866245.382062:0:69871:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 00000000751322ba.
00000800:00000010:1.0:1787866245.382090:0:69871:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 0000000080f667c4.
00000800:00000010:1.0:1787866245.382098:0:69871:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000f7024cbb.
00000800:00000010:1.0:1787866245.382133:0:69871:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000f7024cbb.
00000400:00000200:0.0:1787866245.382824:0:69879:0:(acceptor.c:578:lnet_acceptor()) Accepted connection on ens2:988
00000400:00000200:0.0:1787866245.382887:0:69879:0:(acceptor.c:502:lnet_accept()) Accept 192.168.203.58@tcp from 255.3.0.2
00000800:00000010:0.0:1787866245.382892:0:69879:0:(socklnd.c:737:ksocknal_accept()) alloc '(cr)': 32 at 0000000021f78ebd.
00000800:00000010:1.0:1787866245.383560:0:69872:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 00000000a83d358f.
00000800:00000010:1.0:1787866245.383569:0:69872:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 000000002c79d9df.
00000800:00000010:1.0:1787866245.383591:0:69872:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 00000000b65b4a7c.
00000800:00000200:1.1:1787866245.383598:0:69872:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000065f3099b] -> 12345-192.168.203.58@tcp (2)
00000800:00000200:1.1:1787866245.383603:0:69872:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.203.58@tcp 00000000b65b4a7c deleted
00000800:00000010:1.1:1787866245.383605:0:69872:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 00000000b65b4a7c.
00000800:00000200:1.1:1787866245.383610:0:69872:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1
00000800:00000200:1.1:1787866245.383619:0:69872:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000200:1.1:1787866245.383625:0:69872:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000050d5d605 type 2, nob 96 niov 1 nkiov 0
00000800:00000200:1.0:1787866245.383631:0:69872:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.203.58@tcp p 3.x 192.168.203.58 -> 192.168.203.58:1023 incarnation:1787866243074407332 sched[0]
00000800:00000010:1.0:1787866245.383637:0:69872:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000f7024cbb.
00000800:00000010:1.0:1787866245.383946:0:69872:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000f7024cbb.
00000800:00000010:1.0:1787866245.383952:0:69872:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 000000002c79d9df.
00000800:00000200:0.1:1787866245.384052:0:69871:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1
00000800:00000200:0.0:1787866245.384059:0:69871:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.203.58@tcp p 3.x 192.168.203.58 -> 192.168.203.58:988 incarnation:1787866243074407332 sched[0]
00000800:00000010:0.0:1787866245.384063:0:69871:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 0000000080f667c4.
00000400:00000200:0.0:1787866245.393461:0:69871:0:(lib-socket.c:438:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.203.58:988
00000800:00000010:1.0:1787866245.395924:0:69872:0:(socklnd_cb.c:2179:ksocknal_connd()) kfreed 'cr': 32 at 0000000021f78ebd.
00000800:00000200:1.0:1787866245.396173:0:69877:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1787866245.396178:0:69877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000050d5d605.
00000400:00000200:1.0:1787866245.396184:0:69877:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1787866245.396195:0:69877:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: GET: OK
00000400:00000200:1.0:1787866245.396202:0:69877:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 5
00000400:00000200:1.0:1787866245.396206:0:69877:0:(peer.c:3241:lnet_discovery_event_send()) Ping Send to 192.168.203.58@tcp: 0
00000400:00000200:1.0:1787866245.396238:0:69877:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : GET - for me
00000400:00000200:1.0:1787866245.396273:0:69877:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.58@tcp of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:1.0:1787866245.396291:0:69877:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.203.58@tcp of length 64/272 into md 0xd [1] + 0
00000400:00000200:1.0:1787866245.396308:0:69877:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.203.58@tcp to NMR:  192.168.203.58@tcp local destination
00000400:00000200:1.0:1787866245.396314:0:69877:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [33:33:256:4294967295] 192.168.203.58@tcp LPNI seq info [2:2:8:4294967295]
00000400:00000200:1.0:1787866245.396325:0:69877:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) 192.168.203.58@tcp : REPLY try# 0
00000800:00000200:1.0:1787866245.396331:0:69877:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.203.58@tcp
00000800:00000010:1.0:1787866245.396337:0:69877:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002b1d620e.
00000800:00000200:1.0:1787866245.396344:0:69877:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000065f3099b] -> 12345-192.168.203.58@tcp (4)
00000800:00000200:1.0:1787866245.396351:0:69877:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000200:1.0:1787866245.396356:0:69877:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002b1d620e type 3, nob 160 niov 1 nkiov 1
00000800:00000200:1.0:1787866245.396414:0:69877:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1787866245.396416:0:69877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000002b1d620e.
00000400:00000200:1.0:1787866245.396420:0:69877:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1787866245.396424:0:69877:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: REPLY: OK
00000400:00000200:1.0:1787866245.396440:0:69877:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : REPLY - for me
00000400:00000200:1.0:1787866245.396449:0:69877:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.203.58@tcp: Reply from 12345-192.168.203.58@tcp of length 64/64 into md 0x11
00000400:00000200:1.0:1787866245.396456:0:69877:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1787866245.396459:0:69877:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: REPLY: OK
00000400:00000200:1.0:1787866245.396464:0:69877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c3a7df46
00000400:00000010:1.0:1787866245.396467:0:69877:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000c3a7df46.
00000400:00000200:1.0:1787866245.396470:0:69877:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 3
00000400:00000200:1.0:1787866245.396473:0:69877:0:(peer.c:3119:lnet_discovery_event_reply()) Peer 192.168.203.58@tcp has discovery enabled
00000400:00000200:1.0:1787866245.396477:0:69877:0:(peer.c:3145:lnet_discovery_event_reply()) peer 192.168.203.58@tcp(00000000ac559109) is MR capable
00000400:00000200:1.0:1787866245.396480:0:69877:0:(peer.c:1196:lnet_peer_ni_clr_non_mr_pref_nid()) peer 192.168.203.58@tcp: 0
00000400:00000200:1.0:1787866245.396484:0:69877:0:(peer.c:3200:lnet_discovery_event_reply()) peer 192.168.203.58@tcp data present 12. state = 0x241
00000400:00000200:1.0:1787866245.396489:0:69877:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.203.58@tcp: -114
00000400:00000200:1.0:1787866245.396772:0:69867:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0
00000400:00000200:1.0:1787866245.396780:0:69867:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000ac559109) state 0xc1
00000400:00000010:1.0:1787866245.396789:0:69867:0:(peer.c:3474:lnet_peer_merge_data()) alloc '(curnis)': 60 at 00000000f7024cbb.
00000400:00000010:1.0:1787866245.396793:0:69867:0:(peer.c:3475:lnet_peer_merge_data()) alloc '(addnis)': 72 at 000000003482d2b6.
00000400:00000010:1.0:1787866245.396801:0:69867:0:(peer.c:3476:lnet_peer_merge_data()) alloc '(delnis)': 60 at 0000000038afe6ac.
00000400:00000010:1.0:1787866245.396812:0:69867:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 000000004d833941.
00000400:00000200:1.0:1787866245.396818:0:69867:0:(peer.c:184:lnet_peer_ni_alloc()) 000000004d833941 nid 192.168.203.58@tcp1
00000400:00000010:1.0:1787866245.396822:0:69867:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 00000000402cef62.
00000400:00000200:1.0:1787866245.396824:0:69867:0:(peer.c:203:lnet_peer_net_alloc()) 00000000402cef62 net tcp1
00000400:00000200:1.0:1787866245.396862:0:69867:0:(peer.c:1804:lnet_peer_attach_peer_ni()) peer 192.168.203.58@tcp NID 192.168.203.58@tcp1 flags 0x11
00000400:00000010:1.0:1787866245.396867:0:69867:0:(peer.c:3623:lnet_peer_merge_data()) kfreed 'curnis': 60 at 00000000f7024cbb.
00000400:00000010:1.0:1787866245.396871:0:69867:0:(peer.c:3624:lnet_peer_merge_data()) kfreed 'addnis': 72 at 000000003482d2b6.
00000400:00000010:1.0:1787866245.396873:0:69867:0:(peer.c:3625:lnet_peer_merge_data()) kfreed 'delnis': 60 at 0000000038afe6ac.
00000400:00000010:1.0:1787866245.396876:0:69867:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000033cb82bd.
00000400:00000200:1.0:1787866245.396880:0:69867:0:(peer.c:3628:lnet_peer_merge_data()) peer 192.168.203.58@tcp (00000000ac559109): 0
00000400:00000200:1.0:1787866245.396883:0:69867:0:(peer.c:3940:lnet_peer_data_present()) peer 192.168.203.58@tcp(00000000ac559109): 0. state = 0x141
00000400:00000200:1.0:1787866245.396887:0:69867:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000ac559109) state 0x141 rc 1
00000400:00000200:1.0:1787866245.396890:0:69867:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000ac559109) state 0x141
00000400:00000010:1.0:1787866245.396909:0:69867:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006f8af1d3.
00000400:00000010:1.0:1787866245.396917:0:69867:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000eefdcd83.
00000400:00000200:1.0:1787866245.396921:0:69867:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.58@tcp
00000400:00000200:1.0:1787866245.396928:0:69867:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.203.58@tcp to MR:  192.168.203.58@tcp local destination
00000400:00000200:1.0:1787866245.396933:0:69867:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [34:34:256:4294967295] 192.168.203.58@tcp LPNI seq info [3:3:8:4294967295]
00000400:00000200:1.0:1787866245.396938:0:69867:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:1.0:1787866245.396943:0:69867:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787866245.396947:0:69867:0:(socklnd_cb.c:931:ksocknal_send()) sending 64 bytes in 1 frags to 12345-192.168.203.58@tcp
00000800:00000010:1.0:1787866245.396950:0:69867:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002b1d620e.
00000800:00000200:1.0:1787866245.396954:0:69867:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000065f3099b] -> 12345-192.168.203.58@tcp (4)
00000800:00000200:1.0:1787866245.396958:0:69867:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000200:1.0:1787866245.396961:0:69867:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002b1d620e type 1, nob 160 niov 1 nkiov 1
00000400:00000200:1.0:1787866245.396970:0:69867:0:(peer.c:4164:lnet_peer_send_push()) peer 192.168.203.58@tcp
00000400:00000200:1.0:1787866245.396973:0:69867:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000ac559109) state 0x541 rc 0
00000800:00000200:1.0:1787866245.397029:0:69877:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1787866245.397032:0:69877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000002b1d620e.
00000400:00000200:1.0:1787866245.397034:0:69877:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1787866245.397044:0:69877:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: PUT: OK
00000400:00000200:1.0:1787866245.397047:0:69877:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 5
00000400:00000200:1.0:1787866245.397049:0:69877:0:(peer.c:3241:lnet_discovery_event_send()) Push Send to 192.168.203.58@tcp: 0
00000400:00000200:1.0:1787866245.397062:0:69877:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : PUT - for me
00000400:00000200:1.0:1787866245.397082:0:69877:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.58@tcp of length 64 into portal 0 MB=0x8000000000000000
00000400:00000200:1.0:1787866245.397087:0:69877:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 0 from 12345-192.168.203.58@tcp of length 64/64 into md 0x5 [1] + 0
00000400:00000010:1.0:1787866245.397092:0:69877:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000076084a9f.
00000400:00000200:1.0:1787866245.397097:0:69877:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000006e6648de
00000400:00000200:1.0:1787866245.397104:0:69877:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1787866245.397108:0:69877:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: PUT: OK
00000400:00000200:1.0:1787866245.397111:0:69877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006e6648de
00000400:00000010:1.0:1787866245.397113:0:69877:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000006e6648de.
00000400:00000200:1.0:1787866245.397117:0:69877:0:(api-ni.c:2393:lnet_push_target_event_handler()) type 2 status 0 unlinked 1
00000400:00000200:1.0:1787866245.397124:0:69877:0:(peer.c:2778:lnet_peer_push_event()) peer 192.168.203.58@tcp(00000000ac559109) is MR
00000400:00000010:1.0:1787866245.397128:0:69877:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 73 at 000000003482d2b6.
00000400:00000200:1.0:1787866245.397132:0:69877:0:(peer.c:2852:lnet_peer_push_event()) Received Push 192.168.203.58@tcp 12
00000400:00000200:1.0:1787866245.397135:0:69877:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.203.58@tcp: -114
00000400:00000200:1.0:1787866245.397146:0:69877:0:(lib-md.c:96:lnet_cpt_of_md()) Couldn't resolve first page of md 0000000000000000 with offset 0
00000400:00000200:1.0:1787866245.397151:0:69877:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.203.58@tcp to MR:  192.168.203.58@tcp local destination
00000400:00000200:1.0:1787866245.397155:0:69877:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [35:35:256:4294967295] 192.168.203.58@tcp LPNI seq info [4:4:8:4294967295]
00000400:00000200:1.0:1787866245.397162:0:69877:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) 192.168.203.58@tcp : ACK try# 0
00000800:00000200:1.0:1787866245.397167:0:69877:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 1 frags to 12345-192.168.203.58@tcp
00000800:00000010:1.0:1787866245.397170:0:69877:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002b1d620e.
00000800:00000200:1.0:1787866245.397175:0:69877:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000065f3099b] -> 12345-192.168.203.58@tcp (4)
00000800:00000200:1.0:1787866245.397178:0:69877:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000200:1.0:1787866245.397180:0:69877:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002b1d620e type 0, nob 96 niov 1 nkiov 0
00000800:00000200:1.0:1787866245.397233:0:69877:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1787866245.397235:0:69877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000002b1d620e.
00000400:00000200:1.0:1787866245.397238:0:69877:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1787866245.397241:0:69877:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: ACK: OK
00000400:00000200:1.0:1787866245.397259:0:69877:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp) <- 192.168.203.58@tcp : ACK - for me
00000400:00000200:1.0:1787866245.397266:0:69877:0:(lib-move.c:4539:lnet_parse_ack()) 192.168.203.58@tcp: ACK from 12345-192.168.203.58@tcp into md 0x15
00000400:00000200:1.0:1787866245.397269:0:69877:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1787866245.397272:0:69877:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: ACK: OK
00000400:00000200:1.0:1787866245.397274:0:69877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006f8af1d3
00000400:00000010:1.0:1787866245.397276:0:69877:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000006f8af1d3.
00000400:00000200:1.0:1787866245.397278:0:69877:0:(peer.c:3289:lnet_discovery_event_handler()) Received event: 4
00000400:00000200:1.0:1787866245.397280:0:69877:0:(peer.c:3007:lnet_discovery_event_ack()) peer 192.168.203.58@tcp ev->status 0
00000400:00000200:1.0:1787866245.397283:0:69877:0:(peer.c:2613:lnet_peer_queue_for_discovery()) Queue peer 192.168.203.58@tcp: -114
00000400:00000200:1.0:1787866245.397293:0:69867:0:(peer.c:4230:lnet_peer_discovery_wait_for_work()) woken: 0
00000400:00000010:1.0:1787866245.397298:0:69867:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000cb4b1f32.
00000400:00000010:1.0:1787866245.397301:0:69867:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006f8af1d3.
00000400:00000200:1.0:1787866245.397306:0:69867:0:(api-ni.c:2383:lnet_push_target_post()) posted push target 00000000e571f8f2
00000400:00000200:1.0:1787866245.397310:0:69867:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000ac559109) state 0xc1
00000400:00000010:1.0:1787866245.397313:0:69867:0:(peer.c:3474:lnet_peer_merge_data()) alloc '(curnis)': 60 at 0000000038afe6ac.
00000400:00000010:1.0:1787866245.397315:0:69867:0:(peer.c:3475:lnet_peer_merge_data()) alloc '(addnis)': 72 at 00000000f82fc80e.
00000400:00000010:1.0:1787866245.397316:0:69867:0:(peer.c:3476:lnet_peer_merge_data()) alloc '(delnis)': 60 at 00000000f7024cbb.
00000400:00000010:1.0:1787866245.397319:0:69867:0:(peer.c:3623:lnet_peer_merge_data()) kfreed 'curnis': 60 at 0000000038afe6ac.
00000400:00000010:1.0:1787866245.397322:0:69867:0:(peer.c:3624:lnet_peer_merge_data()) kfreed 'addnis': 72 at 00000000f82fc80e.
00000400:00000010:1.0:1787866245.397324:0:69867:0:(peer.c:3625:lnet_peer_merge_data()) kfreed 'delnis': 60 at 00000000f7024cbb.
00000400:00000010:1.0:1787866245.397326:0:69867:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 73 at 000000003482d2b6.
00000400:00000200:1.0:1787866245.397328:0:69867:0:(peer.c:3628:lnet_peer_merge_data()) peer 192.168.203.58@tcp (00000000ac559109): 0
00000400:00000200:1.0:1787866245.397330:0:69867:0:(peer.c:3940:lnet_peer_data_present()) peer 192.168.203.58@tcp(00000000ac559109): 0. state = 0x141
00000400:00000200:1.0:1787866245.397333:0:69867:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000ac559109) state 0x141 rc 1
00000400:00000200:1.0:1787866245.397335:0:69867:0:(peer.c:4342:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000ac559109) state 0x141
00000400:00000200:1.0:1787866245.397338:0:69867:0:(peer.c:4082:lnet_peer_discovered()) peer 192.168.203.58@tcp
00000400:00000200:1.0:1787866245.397339:0:69867:0:(peer.c:4360:lnet_peer_discovery()) peer 192.168.203.58@tcp(00000000ac559109) state 0x111 rc 0
00000400:00000200:1.0:1787866245.397342:0:69867:0:(peer.c:2629:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 192.168.203.58@tcp
00000400:00000200:1.0:1787866245.397390:0:70118:0:(peer.c:2984:lnet_discover_peer_locked()) peer 192.168.203.58@tcp NID 192.168.203.58@tcp: 0. discovery complete
00000800:00000010:0.0:1787866245.398094:0:69871:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 000000002910fc84.
00000800:00000010:0.0:1787866245.398103:0:69871:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 00000000915532e6.
00000800:00000010:0.0:1787866245.398111:0:69871:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000280e6db6.
00000400:00000200:3.0:1787866245.398124:0:69879:0:(acceptor.c:578:lnet_acceptor()) Accepted connection on ens2:988
00000800:00000010:0.0:1787866245.398155:0:69871:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000280e6db6.
00000400:00000200:3.0:1787866245.399053:0:69879:0:(acceptor.c:502:lnet_accept()) Accept 192.168.203.58@tcp from 254.3.0.2
00000800:00000010:3.0:1787866245.399060:0:69879:0:(socklnd.c:737:ksocknal_accept()) alloc '(cr)': 32 at 00000000f057acd7.
00000800:00000010:0.0:1787866245.399615:0:69873:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at 0000000032af9657.
00000800:00000010:0.0:1787866245.399624:0:69873:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 00000000c313c745.
00000800:00000010:0.0:1787866245.399652:0:69873:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 00000000776de25d.
00000800:00000200:0.1:1787866245.399660:0:69873:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000065f3099b] -> 12345-192.168.203.58@tcp (4)
00000800:00000200:0.1:1787866245.399666:0:69873:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.203.58@tcp 00000000776de25d deleted
00000800:00000010:0.1:1787866245.399668:0:69873:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 00000000776de25d.
00000800:00000200:0.1:1787866245.399685:0:69873:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 3, ksnr_connected a ksnr_max_conns 1
00000800:00000200:0.0:1787866245.399693:0:69873:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.203.58@tcp p 3.x 192.168.203.58 -> 192.168.203.58:1022 incarnation:1787866243074407332 sched[0]
00000800:00000010:0.0:1787866245.399700:0:69873:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000280e6db6.
00000800:00000010:0.0:1787866245.399829:0:69873:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000280e6db6.
00000800:00000010:0.0:1787866245.399849:0:69873:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 00000000c313c745.
00000800:00000010:0.0:1787866245.399878:0:69873:0:(socklnd_cb.c:2179:ksocknal_connd()) kfreed 'cr': 32 at 00000000f057acd7.
00000800:00000200:2.1:1787866245.399886:0:69871:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 2, ksnr_connected e ksnr_max_conns 1
00000800:00000200:2.0:1787866245.399894:0:69871:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.203.58@tcp p 3.x 192.168.203.58 -> 192.168.203.58:988 incarnation:1787866243074407332 sched[0]
00000800:00000010:2.0:1787866245.399909:0:69871:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at 00000000915532e6.
00000001:02000400:2.0:1787866245.421484:0:70119:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set recovery_limit 0
00000400:00000010:2.0:1787866245.489701:0:70122:0:(module.c:388:lnet_ioctl_getdata()) alloc '(*hdr_pp)': 12 at 0000000026fefa59.
00000400:00000010:2.0:1787866245.489720:0:70122:0:(module.c:494:lnet_psdev_ioctl()) kfreed 'hdr': 12 at 0000000026fefa59.
00000001:02000400:3.0:1787866245.541248:0:70124:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl set max_recovery_ping_interval 4
00000001:02000400:2.0:1787866245.575893:0:70126:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp -d *@tcp -m GET -r 1 -e remote_error
00000400:00000200:1.0:1787866245.605755:0:70127:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866245.605770:0:70127:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866245.605785:0:70127:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp
00000400:00000200:1.0:1787866245.605787:0:70127:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866245.605796:0:70127:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp
00000400:00000200:1.0:1787866245.605797:0:70127:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866245.605799:0:70127:0:(api-ni.c:3207:nla_strnid()) <?> -> <?>
00000400:00000200:1.0:1787866245.605800:0:70127:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866245.605801:0:70127:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866245.605803:0:70127:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866245.605804:0:70127:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866245.605805:0:70127:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866245.605807:0:70127:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866245.605809:0:70127:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000010:1.0:1787866245.605814:0:70127:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 0000000091492709.
00000400:00000200:1.0:1787866245.605823:0:70127:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp, dst 255.255.255.255@tcp, rate 1, interval 0
00000001:02000400:0.0:1787866245.625628:0:70128:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl fault drop add -s *@tcp1 -d *@tcp1 -m GET -r 1 -e remote_error
00000400:00000200:3.0:1787866245.668508:0:70129:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866245.668523:0:70129:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866245.668540:0:70129:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1
00000400:00000200:3.0:1787866245.668542:0:70129:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866245.668546:0:70129:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1
00000400:00000200:3.0:1787866245.668547:0:70129:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866245.668550:0:70129:0:(api-ni.c:3207:nla_strnid()) <?> -> <?>
00000400:00000200:3.0:1787866245.668551:0:70129:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866245.668552:0:70129:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866245.668554:0:70129:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866245.668555:0:70129:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866245.668557:0:70129:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866245.668677:0:70129:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:3.0:1787866245.668679:0:70129:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000010:3.0:1787866245.668687:0:70129:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 0000000027824e9b.
00000400:00000200:3.0:1787866245.668697:0:70129:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp1, dst 255.255.255.255@tcp1, rate 1, interval 0
00000001:02000400:2.0:1787866245.692294:0:70130:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net_drop add -s *@tcp -d *@tcp -r 1
00000400:00000200:0.0:1787866245.727305:0:70131:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866245.727319:0:70131:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866245.727333:0:70131:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp
00000400:00000200:0.0:1787866245.727335:0:70131:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866245.727337:0:70131:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp -> 255.255.255.255@tcp
00000400:00000200:0.0:1787866245.727338:0:70131:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866245.727339:0:70131:0:(api-ni.c:3207:nla_strnid()) <?> -> <?>
00000400:00000200:0.0:1787866245.727340:0:70131:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866245.727341:0:70131:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866245.727342:0:70131:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866245.727343:0:70131:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866245.727344:0:70131:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866245.727345:0:70131:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:0.0:1787866245.727347:0:70131:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000010:0.0:1787866245.727350:0:70131:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 000000005e42321f.
00000400:00000200:0.0:1787866245.727358:0:70131:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp, dst 255.255.255.255@tcp, rate 1, interval 0
00000001:02000400:0.0:1787866245.742467:0:70132:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl net_drop add -s *@tcp1 -d *@tcp1 -r 1
00000400:00000200:1.0:1787866245.769509:0:70133:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866245.769523:0:70133:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866245.769541:0:70133:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1
00000400:00000200:1.0:1787866245.769543:0:70133:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866245.769546:0:70133:0:(api-ni.c:3207:nla_strnid()) 255.255.255.255@tcp1 -> 255.255.255.255@tcp1
00000400:00000200:1.0:1787866245.769548:0:70133:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866245.769550:0:70133:0:(api-ni.c:3207:nla_strnid()) <?> -> <?>
00000400:00000200:1.0:1787866245.769551:0:70133:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866245.769552:0:70133:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866245.769554:0:70133:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866245.769556:0:70133:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866245.769557:0:70133:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866245.769559:0:70133:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000200:1.0:1787866245.769561:0:70133:0:(api-ni.c:10008:lnet_fault_cmd()) attr type: 5
00000400:00000010:1.0:1787866245.769566:0:70133:0:(net_fault.c:198:lnet_drop_rule_add()) alloc '(rule)': 312 at 00000000a9f9657c.
00000400:00000200:1.0:1787866245.769577:0:70133:0:(net_fault.c:227:lnet_drop_rule_add()) Added drop rule: src 255.255.255.255@tcp1, dst 255.255.255.255@tcp1, rate 1, interval 0
00000001:02000400:3.0:1787866245.808455:0:70134:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl peer set --nid 192.168.203.58@tcp --health 0
00000400:00000200:0.0:1787866245.852519:0:70135:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.203.58@tcp added to recovery queue. ping count: 0 next ping: 1456 last alive: 1455 health: 0
00000400:00000010:3.0:1787866245.955227:0:70141:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 000000000264b128.
00000400:00000010:3.0:1787866245.955323:0:70141:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 000000000264b128.
00000400:00000010:1.0:1787866247.111608:0:70152:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 00000000a65c2bd9.
00000400:00000010:1.0:1787866247.113466:0:70152:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 00000000a65c2bd9.
00000400:00000010:3.0:1787866247.123936:0:69866:0:(lib-move.c:3721:lnet_recover_peer_nis()) alloc '(ev_info)': 24 at 00000000c8985263.
00000400:00000010:3.0:1787866247.123957:0:69866:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000079ed8d7e.
00000400:00000010:3.0:1787866247.123964:0:69866:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f6712e49.
00000400:00000010:3.0:1787866247.123996:0:69866:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000e3c501af.
00000400:00000200:3.0:1787866247.124009:0:69866:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.203.58@tcp
00000400:00000200:3.0:1787866247.124027:0:69866:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR:  192.168.203.58@tcp local destination
00000400:00000200:3.0:1787866247.124035:0:69866:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.58@tcp
00000400:00000200:3.0:1787866247.124040:0:69866:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [36:36:256:4294967295] 192.168.203.58@tcp LPNI seq info [5:5:8:4294967295]
00000400:00000200:3.0:1787866247.124048:0:69866:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:3.0:1787866247.124074:0:69866:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:<?>) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : GET try# 0
00000800:00000200:3.0:1787866247.124081:0:69866:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.203.58@tcp
00000800:00000010:3.0:1787866247.124088:0:69866:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000087d0ec8d.
00000800:00000200:3.0:1787866247.124095:0:69866:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000065f3099b] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:3.0:1787866247.124105:0:69866:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000200:3.0:1787866247.124109:0:69866:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000087d0ec8d type 2, nob 96 niov 1 nkiov 0
00000400:00000200:0.0:1787866247.124189:0:69876:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 1
00000400:00000200:0.0:1787866247.124199:0:69876:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.203.58@tcp(192.168.203.58@tcp)->dst 192.168.203.58@tcp: GET simulate health error: REMOTE_ERROR
00000800:02000400:0.0:1787866247.124202:0:69876:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.203.58: rc = -22
00000400:00000200:3.0:1787866247.124243:0:69866:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.203.58@tcp added to recovery queue. ping count: 1 next ping: 1458 last alive: 1455 health: 0
00000800:00000200:0.0:1787866247.134191:0:69876:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [00000000751322ba] Error -22 on write to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000010:0.0:1787866247.134203:0:69876:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000087d0ec8d.
00000400:00000200:0.0:1787866247.134211:0:69876:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787866247.134217:0:69876:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: GET: REMOTE_ERROR
00000400:00000200:0.0:1787866247.134229:0:69876:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5
00000400:00000200:0.0:1787866247.134233:0:69876:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.203.58@tcp recovery message sent unsuccessfully:-5
00000400:00000200:0.0:1787866247.134239:0:69876:0:(lib-move.c:4104:lnet_handle_recovery_reply()) peer NI (192.168.203.58@tcp) recovery failed with -5
00000400:00000200:3.0:1787866248.147092:0:69866:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f6712e49
00000400:00000010:3.0:1787866248.147110:0:69866:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000f6712e49.
00000400:00000200:3.0:1787866248.147121:0:69866:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0
00000400:00000200:3.0:1787866248.147130:0:69866:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.203.58@tcp recovery ping unlinked
00000400:00000010:3.0:1787866248.147138:0:69866:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000c8985263.
00000400:00000010:3.0:1787866248.147145:0:69866:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000079ed8d7e.
00000400:00000010:3.0:1787866248.147159:0:69866:0:(lib-move.c:3721:lnet_recover_peer_nis()) alloc '(ev_info)': 24 at 00000000c8985263.
00000400:00000010:3.0:1787866248.147165:0:69866:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000079ed8d7e.
00000400:00000010:3.0:1787866248.147170:0:69866:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f6712e49.
00000400:00000010:3.0:1787866248.147177:0:69866:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000d11fca45.
00000400:00000200:3.0:1787866248.147181:0:69866:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.203.58@tcp
00000400:00000200:3.0:1787866248.147194:0:69866:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR:  192.168.203.58@tcp local destination
00000400:00000200:3.0:1787866248.147202:0:69866:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.58@tcp
00000400:00000200:3.0:1787866248.147206:0:69866:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [37:37:256:4294967295] 192.168.203.58@tcp LPNI seq info [6:6:8:4294967295]
00000400:00000200:3.0:1787866248.147213:0:69866:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:3.0:1787866248.147220:0:69866:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:<?>) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : GET try# 0
00000800:00000200:3.0:1787866248.147225:0:69866:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.203.58@tcp
00000800:00000010:3.0:1787866248.147233:0:69866:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000087d0ec8d.
00000800:00000200:3.0:1787866248.147240:0:69866:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000065f3099b] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:3.0:1787866248.147250:0:69866:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000200:3.0:1787866248.147256:0:69866:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000087d0ec8d type 2, nob 96 niov 1 nkiov 0
00000400:00000200:3.0:1787866248.147334:0:69866:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.203.58@tcp added to recovery queue. ping count: 2 next ping: 1462 last alive: 1455 health: 0
00000400:00000200:0.0:1787866248.147594:0:69878:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 2
00000400:00000200:0.0:1787866248.147602:0:69878:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.203.58@tcp(192.168.203.58@tcp)->dst 192.168.203.58@tcp: GET simulate health error: REMOTE_ERROR
00000800:02000400:0.0:1787866248.147606:0:69878:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.203.58: rc = -22
00000800:00000200:0.0:1787866248.147610:0:69878:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [00000000a83d358f] Error -22 on write to 12345-192.168.203.58@tcp ip 192.168.203.58:1023
00000800:00000010:0.0:1787866248.147616:0:69878:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000087d0ec8d.
00000400:00000200:0.0:1787866248.147622:0:69878:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787866248.147626:0:69878:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: GET: REMOTE_ERROR
00000400:00000200:0.0:1787866248.147633:0:69878:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5
00000400:00000200:0.0:1787866248.147636:0:69878:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.203.58@tcp recovery message sent unsuccessfully:-5
00000400:00000200:0.0:1787866248.147639:0:69878:0:(lib-move.c:4104:lnet_handle_recovery_reply()) peer NI (192.168.203.58@tcp) recovery failed with -5
00000400:00000010:2.0:1787866248.262232:0:70163:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 0000000026fefa59.
00000400:00000010:2.0:1787866248.262345:0:70163:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 0000000026fefa59.
00000400:00000200:3.0:1787866249.171075:0:69866:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f6712e49
00000400:00000010:3.0:1787866249.171114:0:69866:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 00000000f6712e49.
00000400:00000200:3.0:1787866249.171124:0:69866:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0
00000400:00000200:3.0:1787866249.171132:0:69866:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.203.58@tcp recovery ping unlinked
00000400:00000010:3.0:1787866249.171140:0:69866:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000c8985263.
00000400:00000010:3.0:1787866249.171146:0:69866:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000079ed8d7e.
00000400:00000010:0.0:1787866249.393436:0:70174:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 00000000ac951521.
00000400:00000010:0.0:1787866249.393554:0:70174:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 00000000ac951521.
00000400:00000010:2.0:1787866250.509189:0:70180:0:(api-ni.c:7707:lnet_peer_ni_show_start()) alloc '(plist)': 16 at 0000000026fefa59.
00000400:00000010:2.0:1787866250.509303:0:70180:0:(api-ni.c:7684:lnet_peer_ni_show_done()) kfreed 'plist': 16 at 0000000026fefa59.
00000001:02000400:1.0:1787866252.038085:0:70308:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_211: @@@@@@ FAIL: Unexpected ping count
00000400:00000010:0.0:1787866252.242962:0:69866:0:(lib-move.c:3721:lnet_recover_peer_nis()) alloc '(ev_info)': 24 at 00000000a70d3705.
00000400:00000010:0.0:1787866252.242974:0:69866:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000b2adbc94.
00000400:00000010:0.0:1787866252.242979:0:69866:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000008925a378.
00000400:00000010:0.0:1787866252.242987:0:69866:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a147970c.
00000400:00000200:0.0:1787866252.242995:0:69866:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.203.58@tcp
00000400:00000200:0.0:1787866252.243006:0:69866:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR:  192.168.203.58@tcp local destination
00000400:00000200:0.0:1787866252.243012:0:69866:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.58@tcp
00000400:00000200:0.0:1787866252.243016:0:69866:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.58@tcp NI seq info: [38:38:256:4294967295] 192.168.203.58@tcp LPNI seq info [7:7:8:4294967295]
00000400:00000200:0.0:1787866252.243031:0:69866:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.58@tcp
00000400:00000200:0.0:1787866252.243038:0:69866:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.58@tcp(192.168.203.58@tcp:<?>) -> 192.168.203.58@tcp(192.168.203.58@tcp:192.168.203.58@tcp) <?> : GET try# 0
00000800:00000200:0.0:1787866252.243043:0:69866:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.203.58@tcp
00000800:00000010:0.0:1787866252.243048:0:69866:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000b393dae5.
00000800:00000200:0.0:1787866252.243054:0:69866:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000065f3099b] -> 12345-192.168.203.58@tcp (6)
00000800:00000200:0.0:1787866252.243061:0:69866:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000200:0.0:1787866252.243065:0:69866:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000b393dae5 type 2, nob 96 niov 1 nkiov 0
00000400:00000200:0.0:1787866252.243082:0:69866:0:(peer.c:4768:lnet_peer_ni_add_to_recoveryq_locked()) 192.168.203.58@tcp added to recovery queue. ping count: 3 next ping: 1466 last alive: 1455 health: 0
00000400:00000200:1.0:1787866252.243812:0:69877:0:(net_fault.c:505:drop_rule_match()) Drop Rule 255.255.255.255@tcp->255.255.255.255@tcp: next drop: 3
00000400:00000200:1.0:1787866252.243819:0:69877:0:(lib-msg.c:1176:lnet_send_error_simulation()) src 192.168.203.58@tcp(192.168.203.58@tcp)->dst 192.168.203.58@tcp: GET simulate health error: REMOTE_ERROR
00000800:02000400:1.0:1787866252.243823:0:69877:0:(socklnd_cb.c:536:ksocknal_process_transmit()) There was an unexpected network error while writing to 192.168.203.58: rc = -22
00000800:00000200:1.0:1787866252.243827:0:69877:0:(socklnd_cb.c:541:ksocknal_process_transmit()) [00000000751322ba] Error -22 on write to 12345-192.168.203.58@tcp ip 192.168.203.58:988
00000800:00000010:1.0:1787866252.243863:0:69877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000b393dae5.
00000400:00000200:1.0:1787866252.243867:0:69877:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787866252.243883:0:69877:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.58@tcp->192.168.203.58@tcp: GET: REMOTE_ERROR
00000400:00000200:1.0:1787866252.243890:0:69877:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: -5
00000400:00000200:1.0:1787866252.243893:0:69877:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.203.58@tcp recovery message sent unsuccessfully:-5
00000400:00000200:1.0:1787866252.243897:0:69877:0:(lib-move.c:4104:lnet_handle_recovery_reply()) peer NI (192.168.203.58@tcp) recovery failed with -5
00000400:00000200:0.0:1787866253.266950:0:69866:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000008925a378
00000400:00000010:0.0:1787866253.266967:0:69866:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at 000000008925a378.
00000400:00000200:0.0:1787866253.266972:0:69866:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 6 status: 0
00000400:00000200:0.0:1787866253.266979:0:69866:0:(lib-move.c:4125:lnet_mt_event_handler()) 192.168.203.58@tcp recovery ping unlinked
00000400:00000010:0.0:1787866253.266985:0:69866:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000a70d3705.
00000400:00000010:0.0:1787866253.266989:0:69866:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000b2adbc94.
Debug log: 4733 lines, 4733 kept, 0 dropped, 0 bad.