From a82081611f92e1db6b66eca0ede51d5e4002b62d Mon Sep 17 00:00:00 2001
From: evm <evm.ftw@gmail.com>
Date: Wed, 30 Oct 2019 14:44:19 -0400
Subject: [PATCH] Make LFA modules contiguous, source file names in .csv output
 for easier comparison between LFA and MC

---
 README           |  11 ++
 basicutils_6x.py | 316 -----------------------------------------------
 cc_base.py       |  15 ++-
 lfa.py           |  15 +--
 maxcut.py        |   6 +-
 5 files changed, 30 insertions(+), 333 deletions(-)
 delete mode 100644 basicutils_6x.py

diff --git a/README b/README
index 6242267..cee63da 100644
--- a/README
+++ b/README
@@ -86,6 +86,10 @@ A couple areas for research:
   - The portion of code that tries to name object files based on common strings
     is completely researchy and open ended.  Lots of things to play with there.
 
+**** MaxCut Parameters & Interpolation ****
+
+  - The only real parameter for MaxCut is a THRESHOLD variable that corresponds to the size at which the algorithm will stop subdividing modules.  A threshold of 4K (0x1000) seems to provide similar sized modules to LFA.  A threshold of 8K (0x2000) seems to be a good upper bound.  A good area of research would be making this not a static cutoff but maybe deciding to stop subdividing based on a connectedness measurement or something along those lines.
+
 **** Output Files ****
 
 CodeCut produces 7 files:
@@ -122,6 +126,13 @@ You can use sfdp to render the graph into a PNG file with a command line like:
 
 sfdp -x -Goverlap=scale -Tpng -Goutputorder=edgesfirst -Nstyle=filled -Nfillcolor=white <target>_lfa_mod_graph.gv > <target>.png
 
+A really nice hierarchical graph can be obtained by adding:
+ranksep=0
+nodesep=0
+to the .gv file and running:
+
+dot -x -Goverlap=scale -Tpng -Goutputorder=edgesfirst -Nstyle=filled -Nfillcolor=white <target>.gv > <target>.png
+
 **** "Canonical" Names ****
 NOTE on IDA and Canonical Names:
 AFAICT IDA doesn't really have a concept of source file / object files in
diff --git a/basicutils_6x.py b/basicutils_6x.py
deleted file mode 100644
index 3070bc4..0000000
--- a/basicutils_6x.py
+++ /dev/null
@@ -1,316 +0,0 @@
-##############################################################################################
-# Copyright 2018 The Johns Hopkins University Applied Physics Laboratory LLC
-# All rights reserved.
-# Permission is hereby granted, free of charge, to any person obtaining a copy of this 
-# software and associated documentation files (the "Software"), to deal in the Software 
-# without restriction, including without limitation the rights to use, copy, modify, 
-# merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 
-# permit persons to whom the Software is furnished to do so.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
-# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR 
-# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 
-# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 
-# OR OTHER DEALINGS IN THE SOFTWARE.
-#
-# HAVE A NICE DAY.
-
-# basicutils - a version-agnostic API for IDA Pro with some (slightly) higher level functionality
-# This is the 6.x version - see basicutils_7x for the 7.x version
-
-import idc
-import struct
-import idautils
-import re
-
-BADADDR = idc.BADADDR
-
-def SegByName(n):
-	start = idc.SegByBase(idc.SegByName(n))
-	if (start != idc.BADADDR):
-		end = idc.SegEnd(start)
-	else:
-		start = idc.BADADDR
-		end = idc.BADADDR
-	return (start,end)
-	
-def GetFunctionName(x):
-	return idc.GetFunctionName(x)
-
-def GetInputFile():
-	return idc.GetInputFile()
-
-def NextFunction(x):
-	return idc.NextFunction(x)
-
-def PrevInstr(ea):
-    return idc.PrevHead(ea, ea-15)
-
-def ForEveryUniqXrefTo( target, fun ):
-    a = 0
-    for xref in idautils.CodeRefsTo(target,0):
-		if idc.GetFunctionAttr(xref,idc.FUNCATTR_START) != a :
-			fun(xref)
-			a = idc.GetFunctionAttr(xref, idc.FUNCATTR_START);
-			
-def ForEveryXrefTo( target, fun ):
-    for xref in idautils.CodeRefsTo(target,0):
-		fun(xref)
-
-def ForEveryUniqXrefToD( target, fun ):
-    a = 0
-    for xref in idautils.CodeRefsTo(target,0):
-		if idc.GetFunctionAttr(xref,idc.FUNCATTR_START) != a :
-			fun(xref, target)
-			a = idc.GetFunctionAttr(xref, idc.FUNCATTR_START);
-		
-def ForEveryXrefToD( target, fun ):
-	for xref in idautils.CodeRefsTo(target,0):
-		fun(xref, target)
-
-def ForEveryFuncInDb( fun ):
-	f = idc.NextFunction(0)
-	while (f != idc.BADADDR):
-		"""print "ev: %#x" % f"""
-		fun(f)
-		f=idc.NextFunction(f)
-
-def NFuncUp( fun, n ) :
-	i=0
-	f=fun
-	while ((i<n) and (f!=idc.BADADDR)):
-		f=idc.PrevFunction(f)
-		i=i+1
-	return f
-	
-def NFuncDown( fun, n ) :
-	i=0
-	f=fun
-	while ((i<n) and (f!=idc.BADADDR)):
-		f=idc.NextFunction(f)
-		i=i+1
-	return f
-
-def FuncMidPt( fun ):
-	fstart = idc.GetFunctionAttr(fun, idc.FUNCATTR_START)
-	fend = idc.GetFunctionAttr(fun, idc.FUNCATTR_END)
-	return fstart+((fend-fstart)/2)
-
-def FuncXrefsFrom ( fun ) :
-	f = set()
-	for item in idautils.FuncItems(fun):
-		for x in idautils.CodeRefsFrom(item,0):
-			s = idc.GetFunctionAttr(x, idc.FUNCATTR_START)
-			if (x == s):
-				f.add(x)
-	#print "func xrefs from"
-	#print f
-	return f
-
-def XrefFromRange ( fun ) :
-	f = FuncXrefsFrom(fun)
-	if f:
-		return (min(f),max(f))
-	else:
-		return (0,0)
-	
-def ProgramAddrRange() :
-	return idc.PrevFunction(idc.BADADDR) - idc.NextFunction(0)
-
-def MemCopy( dest, src, length ) :
-	for i in xrange(0, length):
-		#if (i < 20):
-		#	print "set byte at %#x to %#x" % (dest+i, idc.Byte(src+i))
-		idc.PatchByte(dest+i,idc.Byte(src+i))
-		
-def PrefixRange(start, end, prefix) :
-	x = start
-	while x < end:
-		n = idc.GetFunctionName(x)
-		if n.startswith("sub_"):
-			nn = prefix + n
-			print "Renaming %s to %s\n" % (n, nn)
-			idc.MakeName(x,nn)
-		x = idc.NextFunction(x)
-	
-def snakeToCamelCase(s):
-	f = s.lstrip("_")
-	nf = ""
-	nx = 0
-	x=0
-	while (x<len(f)):
-		#print "%s" % (f[x])
-		if f[x] == '_':
-			nf+=(f[x+1].upper())
-			x+=2
-		else:
-			nf+=f[x]
-			x+=1
-		nx+=1
-	return nf
-	
-def isSnakeCase(s) :
-	p = re.compile("[a-zA-Z0-9]+(_[a-zA-Z0-9]+)+\Z")
-	if p.match(s):
-		return True
-	return False
-	
-#Todo - right now this is going to miss something like FooBARFunction
-def isCamelCase(s) :
-	p = re.compile("([A-Z][a-z0-9]+)([A-Z][a-z0-9]+)+\Z")
-	if p.match(s):
-		return True
-	return False
-	
-#Todo - weed out if it's all uppercase or all uppercase and _, etc.
-
-def isUCSnakeCase(s):
-	p = re.compile("[A-Z0-9]+(_[A-Z0-9]+)+\Z")
-	if p.match(s):
-		return True
-	return False
-
-def isPlausibleFunction(s):
-	if isSnakeCase(s):
-		if isUCSnakeCase(s):
-			return False
-		return True
-	if isCamelCase(s):
-		return True
-	return False
-
-def PrependStrToFuncName(f,s):
-	n = idc.GetFunctionName(f)
-	n = s + n
-	idc.MakeName(f,n)
-
-#The "canonical" name format (for now) is <module name>_<func name>_<address>
-#where <module_name> and <func_name> are in camel case.
-#This is not ideal for a number of reasons but this is a workaround for now
-
-#Return just the "function name" part of the canonical name	
-def GetCanonicalName(f):
-	n = idc.GetFunctionName(f)
-	parts = n.split("_")
-	if len(parts) == 3:
-		return parts[1]
-	else:
-		return None
-
-#Put function in canonical format, given the function name and module name
-def NameCanonical(f,mod_name,func_name):
-	n = "%s_%s_%08x" % (mod_name,func_name,f)
-	print "Renaming %s to %s\n" % (idc.GetFunctionName(f),n)
-	idc.MakeName(f,n)
-
-#Put function in canonical format when it doesn't have a name, but you know the module name
-def RenameFuncWithAddr(f,s):
-	func_name = "unk"
-	NameCanonical(f,s,func_name)
-
-#Use this if you have pre-existing named functions in the DB that are in non-canonical format
-def RenameRangeWithAddr(start,end,s):
-	x = start
-	while (x<=end):
-		n = idc.GetFunctionName(x)
-		if (n.startswith("sub_")):
-			RenameFuncWithAddr(x,s)
-		else:
-			NameCanonical(x,s,n)
-		x = idc.NextFunction(x)
-
-#Rename a function in canonical format without changing the module name
-def CanonicalFuncRename(f,name):
-	n = idc.GetFunctionName(f)
-	parts = n.split("_")
-	new_name = "%s_%s_%08x" % (parts[0],name,f)
-	print "Renaming %s to %s\n" % (n, new_name)
-	idc.MakeName(f,new_name)
-
-#Rename the module name without changing the function name	
-def RenameFuncWithNewMod(f,mod):
-	n = idc.GetFunctionName(f)
-	parts = n.split("_")
-	new_name = "%s_%s_%08x" % (mod,parts[1],f)
-	print "Renaming %s to %s\n" % (n, new_name)
-	idc.MakeName(f,new_name)
-
-#Rename a module (all functions that start with <mod>_)
-def RenameMod(orig, new):
-	i = idc.NextFunction(0)
-	while (i != idc.BADADDR):
-		n = idc.GetFunctionName(i)
-		if n.startswith(orig+"_"):
-			RenameFuncWithNewMod(i,new)
-		i = idc.NextFunction(i)
-
-#Just rename the module over a given range (can be used to split a module and give part a new name)
-def RenameModRange(start, end, new):
-	x = start
-	while (x<=end):
-		n = idc.GetFunctionName(x)
-		RenameFuncWithNewMod(x,new)
-		x = idc.NextFunction(x)
-
-#Given a range of functions, some of which may have names and module names
-# and a module name, put names in canonical format		
-def CanonicalizeRange(start,end,mod):
-	x = start
-	while (x<=end):
-		n = idc.GetFunctionName(x)
-		#if it already starts with mod name, assume it's canonical
-		if (not n.startswith(mod+"_")):
-			if (n.startswith("sub_")):
-				RenameFuncWithAddr(x,mod)
-			#this should be contains "_"
-			elif ("_" in n):
-				n = snakeToCamelCase(n)
-				NameCanonical(x,mod,n)
-			else:
-				NameCanonical(x,mod,n)
-		x = idc.NextFunction(x)	
-
-#Returns a string that is the concatenation of all of the string references from a function, separated by <sep>
-#Iterates through every item in function and looks for data references that are strings    	
-def CompileTextFromFunction(f,sep):
-	s=""
-	faddr = list(idautils.FuncItems(f))
-	for c in range(len(faddr)):
-		for d in idautils.DataRefsFrom(faddr[c]):
-			if idc.GetStringType(d) == 0 and idc.GetString(d):
-				s += " "+ sep + " " + idc.GetString(d)
-	return s
-
-#Returns a string which is the concatenation all of the string references 
-# for an address range in the program, separated by <sep>
-#Similar to above, but iterates over the whole set of functions in the given range    	
-def CompileTextFromRange(start,end,sep):
-	x = start
-	s = ""
-	while (x<=end):
-		#print "Function %x" % x
-		faddr = list(idautils.FuncItems(x))
-		for c in range(len(faddr)):
-			for d in idautils.DataRefsFrom(faddr[c]):
-				#print "Found ref at %x" % faddr[c]
-				if idc.GetStringType(d) == 0 and idc.GetString(d):
-					s += " "+ sep + " " + idc.GetString(d)
-				elif idc.GetStringType(d) == 3 and idc.GetString(d, -1, idc.ASCSTR_UNICODE):
-					s += " " + sep + " " + idc.GetString(d,-1,idc.ASCSTR_UNICODE)
-		x = idc.NextFunction(x)
-	return s
-	
-#Returns a string which is a concatenation of all the function names in the given range
-# separated by <sep>	
-def CompileFuncNamesFromRangeAsText(start,end,sep):
-	x = start
-	s = ""
-	while (x<=end):
-		n = idc.GetFunctionName(x)
-		if (not n.startswith("sub_")):
-			s += " " + sep + " " + n
-		x = idc.NextFunction(x)
-	return s
-
-	
\ No newline at end of file
diff --git a/cc_base.py b/cc_base.py
index 6c32a6c..5b23110 100644
--- a/cc_base.py
+++ b/cc_base.py
@@ -17,6 +17,7 @@
 # HAVE A NICE DAY.
 
 import basicutils_7x as basicutils
+import modnaming
 
 ## CodeCut Basics
 ## A couple of functions for working with function and module lists and outputting results
@@ -114,7 +115,8 @@ def gen_map_file(module_list, suffix):
 	
 	while (c<len(module_list)):
 		m=module_list[c]
-		mlen = basicutils.NextFunction(m.end) - m.start 
+		#mlen = basicutils.NextFunction(m.end) - m.start 
+		mlen = m.end - m.start
 		mlen_str = "0x%x" % mlen
 		file.write("%s0x%016x%s %s\n" % (" .text".ljust(16),m.start,mlen_str.rjust(11),m.name))
 		c+=1
@@ -129,7 +131,7 @@ def print_results(function_list, module_list1, module_list2):
 	file = open(root_name + "_cc_results.csv", "wb")
 	
 	#write header
-	file.write("Function,Function #,LFA Score 1,LFA Score 2,LFA Total,LFA Edge,MC Edge,Function Name,Suggested Mod Name (LFA), Suggested Mod Name(MC)\n");
+	file.write("Function,Function #,LFA Score 1,LFA Score 2,LFA Total,LFA Edge,MC Edge,Function Name,Suggested Mod Name (LFA), Suggested Mod Name(MC),Source Str Ref\n");
 	
 	while (c<len(function_list)):
 		f = function_list[c]
@@ -138,6 +140,13 @@ def print_results(function_list, module_list1, module_list2):
 		m2 = locate_module(module_list2, f.loc)
 		mname1 = m1.name
 		mname2 = m2.name
-		line = "0x%08x, %d , %f, %f, %f, %d, %d, %s, %s, %s\n" % (f.loc,c+1,f.score1, f.score2, f.total_score,f.edge[0],f.edge[1],fname, mname1, mname2)
+		#hacky - should actually find the extent of the function
+		#for now we'll just skip the last one
+		if (c < (len(function_list) - 1)):
+			nf = basicutils.NextFunction(f.loc)
+			func_str_ref, score = modnaming.source_file_strings(f.loc, nf-1)
+		else:
+			func_str_ref=""
+		line = "0x%08x, %d , %f, %f, %f, %d, %d, %s, %s, %s, %s\n" % (f.loc,c+1,f.score1, f.score2, f.total_score,f.edge[0],f.edge[1],fname, mname1, mname2, func_str_ref)
 		file.write(line)
 		c+=1
\ No newline at end of file
diff --git a/lfa.py b/lfa.py
index a08ef7e..3c72698 100644
--- a/lfa.py
+++ b/lfa.py
@@ -210,7 +210,7 @@ def get_last_three(index):
 	i = index-1
 	p=[]
 	while ((c<3) and (i>0)):
-		print "get_last_3: %d,%d" % (c,i)
+		#print "get_last_3: %d,%d" % (c,i)
 		if (g_function_list[i].lfa_skip == 0):
 			p.append(g_function_list[i])
 			c+=1
@@ -225,7 +225,7 @@ def get_lfa_start():
 	c=0;
 	i=0;
 	while (c < 4):
-		print "get_lfa_start: %d,%d" % (c,i)
+		#print "get_lfa_start: %d,%d" % (c,i)
 		if (g_function_list[i].lfa_skip==0):
 			c+=1
 		i+=1
@@ -245,16 +245,11 @@ def edge_detect():
 	c=get_lfa_start()
 	#do edge detection
 	while (c<len(g_function_list)):
-		#TODO: this is not working as previously intended
-		#because the last 3 can have "skipped" entries in them
 		if (g_function_list[c].lfa_skip == 0):
 			f_1,f_2,f_3 = get_last_three(c)
 			p_1 = f_1.total_score
 			p_2 = f_2.total_score
 			p_3 = f_3.total_score
-			#p_1 = g_function_list[c-1].total_score
-			#p_2 = g_function_list[c-2].total_score
-			#p_3 = g_function_list[c-3].total_score
 			s = g_function_list[c].total_score
 			#if score is positive and it is diff of at least 2 from previous
 			#and the previous function was not an edge
@@ -270,8 +265,8 @@ def edge_detect():
 	while(c<len(g_function_list)):
 		f = g_function_list[c]
 		if (f.edge[0] == 1):
-			p = g_function_list[c-1]
-			b_mod = module.bin_module(mod_start,p.loc,0,"")
+			#change from previous code, this will make the modules contiguous
+			b_mod = module.bin_module(mod_start,f.loc-1,0,"")
 			mod_start = f.loc #set the start of the next module to this function (where edge was detected)
 			g_module_list.append(b_mod)
 		c+=1
@@ -294,4 +289,4 @@ def analyze():
 	#Detect edges - object file boundaries
 	edge_detect()
 	
-	return g_function_list, g_module_list
\ No newline at end of file
+	return g_function_list, g_module_list
diff --git a/maxcut.py b/maxcut.py
index bd275c4..e33abc3 100644
--- a/maxcut.py
+++ b/maxcut.py
@@ -119,10 +119,8 @@ def make_cut(region_start, region_end, graph):
 def do_cutting(start, end, graph):
 	nodes = graph.GetNodes()
 	print "do_cutting: start: 0x%x end: 0x%x nodes: 0x%x" % (start, end, nodes)
-	#set this way for simple
-	#THRESHOLD = 0x100
-	#THRESHOLD = 0x1000
-	THRESHOLD = 0x4000
+	THRESHOLD = 0x1000
+	#THRESHOLD = 0x2000
 	
 	if (end - start > THRESHOLD) and (nodes > 1):
 		cut_address = make_cut(start, end,graph)