Merge remote-tracking branch 'origin/GP-2602_ryanmkurtz_macho-libs--SQUASHED'

Ghidra/Features/Base/ghidra_scripts/FixupELFExternalSymbolsScript.java

@@ -1,108 +0,0 @@
-/* ###
- * IP: GHIDRA
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * 
- *      http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-// Fixes up any unresolved external symbols (for ELF binaries).
-//
-// The current program's "External Programs" list needs to be correct before running
-// this script.
-//
-// This script can be run multiple times without harm, generally after updating the "External Programs"
-// list.
-//
-//@category Symbol
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import ghidra.app.script.GhidraScript;
-import ghidra.app.util.importer.MessageLog;
-import ghidra.app.util.opinion.ElfLoader;
-import ghidra.app.util.opinion.Loaded;
-import ghidra.framework.model.*;
-import ghidra.program.model.listing.Library;
-import ghidra.program.model.listing.Program;
-import ghidra.program.util.ELFExternalSymbolResolver;
-import ghidra.util.Msg;
-import ghidra.util.exception.VersionException;
-
-public class FixupELFExternalSymbolsScript extends GhidraScript {
-
-	@Override
-	protected void run() throws Exception {
-		if (!ElfLoader.ELF_NAME.equals(currentProgram.getExecutableFormat())) {
-			Msg.showError(this, null, "FixupELFExternalSymbols",
-				"Current program is not an ELF program!  (" + currentProgram.getExecutableFormat() +
-					")");
-			return;
-		}
-		MessageLog messageLog = new MessageLog();
-		Object consumer = new Object();
-		ProjectData projectData = currentProgram.getDomainFile().getParent().getProjectData();
-		List<Loaded<Program>> loadedPrograms = new ArrayList<>();
-
-		// Add current program to list
-		loadedPrograms.add(new Loaded<>(currentProgram, currentProgram.getName(),
-			currentProgram.getDomainFile().getPathname()));
-
-		// Add external libraries to list
-		for (Library extLibrary : ELFExternalSymbolResolver.getLibrarySearchList(currentProgram)) {
-			monitor.checkCanceled();
-			String libName = extLibrary.getName();
-			String libPath = extLibrary.getAssociatedProgramPath();
-			if (libPath == null) {
-				continue;
-			}
-
-			DomainFile libDomainFile = projectData.getFile(libPath);
-			if (libDomainFile == null) {
-				messageLog.appendMsg("Referenced external program not found: " + libPath);
-				continue;
-			}
-
-			DomainObject libDomainObject = null;
-			try {
-				libDomainObject =
-					libDomainFile.getDomainObject(consumer, false, false, monitor);
-				if (libDomainObject instanceof Program program) {
-					loadedPrograms.add(new Loaded<>(program, libName, libPath));
-				}
-				else {
-					messageLog
-							.appendMsg("Referenced external program is not a program: " + libPath);
-				}
-			}
-			catch (IOException e) {
-				// failed to open library
-				messageLog.appendMsg("Failed to open library dependency project file: " +
-					libDomainFile.getPathname());
-			}
-			catch (VersionException e) {
-				messageLog.appendMsg(
-					"Referenced external program requires updgrade, unable to consider symbols: " +
-						libPath);
-			}
-		}
-
-		// Resolve symbols
-		ELFExternalSymbolResolver.fixUnresolvedExternalSymbols(loadedPrograms, messageLog, monitor);
-
-		// Cleanup
-		for (int i = 1; i < loadedPrograms.size(); i++) {
-			loadedPrograms.get(i).release(consumer);
-		}
-		Msg.info(this, messageLog.toString());
-	}
-
-}

Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/analysis/ExternalSymbolResolverAnalyzer.java

@@ -0,0 +1,130 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.app.plugin.core.analysis;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import ghidra.app.services.*;
+import ghidra.app.util.importer.MessageLog;
+import ghidra.app.util.opinion.*;
+import ghidra.framework.model.*;
+import ghidra.framework.options.Options;
+import ghidra.program.model.address.AddressSetView;
+import ghidra.program.model.listing.Library;
+import ghidra.program.model.listing.Program;
+import ghidra.program.util.ExternalSymbolResolver;
+import ghidra.util.exception.CancelledException;
+import ghidra.util.exception.VersionException;
+import ghidra.util.task.TaskMonitor;
+
+/**
+ * {@link Analyzer} to link unresolved symbols
+ * 
+ * @see ExternalSymbolResolver
+ */
+public class ExternalSymbolResolverAnalyzer extends AbstractAnalyzer {
+
+	private static final String NAME = "External Symbol Resolver";
+	private static final String DESCRIPTION =
+		"Links unresolved external symbols to the first symbol found in the program's required libraries list (found in program properties).";
+
+	/**
+	 * Creates a new {@link MachoFunctionStartsAnalyzer} 
+	 */
+	public ExternalSymbolResolverAnalyzer() {
+		super(NAME, DESCRIPTION, AnalyzerType.BYTE_ANALYZER);
+		setDefaultEnablement(true);
+		setSupportsOneTimeAnalysis();
+
+		// Do it before demangling
+		setPriority(AnalysisPriority.DATA_TYPE_PROPOGATION.before().before().before().before());
+	}
+
+	@Override
+	public boolean canAnalyze(Program program) {
+		Options options = program.getOptions(Program.PROGRAM_INFO);
+		String format = options.getString("Executable Format", null);
+		return ElfLoader.ELF_NAME.equals(format) || MachoLoader.MACH_O_NAME.equals(format);
+	}
+
+	@Override
+	public boolean added(Program program, AddressSetView set, TaskMonitor monitor, MessageLog log)
+			throws CancelledException {
+
+		Object consumer = new Object();
+		log = new MessageLog(); // For now, we don't want the analysis log spammed
+		ProjectData projectData = program.getDomainFile().getParent().getProjectData();
+		List<Loaded<Program>> loadedPrograms = new ArrayList<>();
+
+		// Add program to list
+		loadedPrograms.add(new Loaded<>(program, program.getName(),
+			program.getDomainFile().getParent().getPathname()));
+
+		// Add external libraries to list
+		for (Library extLibrary : ExternalSymbolResolver.getLibrarySearchList(program)) {
+			monitor.checkCancelled();
+			String libPath = extLibrary.getAssociatedProgramPath();
+			if (libPath == null) {
+				continue;
+			}
+
+			DomainFile libDomainFile = projectData.getFile(libPath);
+			if (libDomainFile == null) {
+				log.appendMsg("Referenced external program not found: " + libPath);
+				continue;
+			}
+
+			try {
+				DomainObject libDomainObject =
+					libDomainFile.getDomainObject(consumer, false, false, monitor);
+				if (libDomainObject instanceof Program p) {
+					loadedPrograms.add(new Loaded<>(p, libDomainFile.getName(),
+						libDomainFile.getParent().getPathname()));
+				}
+				else {
+					libDomainObject.release(consumer);
+					log.appendMsg("Referenced external program is not a program: " + libPath);
+				}
+			}
+			catch (IOException e) {
+				log.appendMsg("Failed to open library dependency project file: " +
+					libDomainFile.getPathname());
+			}
+			catch (VersionException e) {
+				log.appendMsg(
+					"Referenced external program requires updgrade, unable to consider symbols: " +
+						libPath);
+			}
+		}
+
+		// Resolve symbols
+		try {
+			ExternalSymbolResolver.fixUnresolvedExternalSymbols(loadedPrograms, false, log,
+				monitor);
+			return true;
+		}
+		catch (IOException e) {
+			return false;
+		}
+		finally {
+			for (int i = 1; i < loadedPrograms.size(); i++) {
+				loadedPrograms.get(i).release(consumer);
+			}
+		}
+	}
+}

Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/ElfLoader.java

@@ -28,7 +28,7 @@ import ghidra.framework.model.Project;
 import ghidra.framework.options.Options;
 import ghidra.program.model.lang.Endian;
 import ghidra.program.model.listing.Program;
-import ghidra.program.util.ELFExternalSymbolResolver;
+import ghidra.program.util.ExternalSymbolResolver;
 import ghidra.util.Msg;
 import ghidra.util.NumericUtilities;
 import ghidra.util.exception.CancelledException;
@@ -47,7 +47,6 @@ public class ElfLoader extends AbstractLibrarySupportLoader {
 	public final static String ELF_ORIGINAL_IMAGE_BASE_PROPERTY = "ELF Original Image Base";
 	public final static String ELF_PRELINKED_PROPERTY = "ELF Prelinked";
 
-	public final static String ELF_REQUIRED_LIBRARY_PROPERTY_PREFIX = "ELF Required Library ["; // followed by "#]"
 	public final static String ELF_SOURCE_FILE_PROPERTY_PREFIX = "ELF Source File ["; // followed by "#]"
 
 	/**
@@ -158,7 +157,8 @@ public class ElfLoader extends AbstractLibrarySupportLoader {
 			throws CancelledException, IOException {
 		super.postLoadProgramFixups(loadedPrograms, project, options, messageLog, monitor);
 
-		ELFExternalSymbolResolver.fixUnresolvedExternalSymbols(loadedPrograms, messageLog, monitor);
+		ExternalSymbolResolver.fixUnresolvedExternalSymbols(loadedPrograms, true, messageLog,
+			monitor);
 	}
 
 	@Override

Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/ElfProgramBuilder.java

@@ -51,6 +51,7 @@ import ghidra.program.model.scalar.Scalar;
 import ghidra.program.model.symbol.*;
 import ghidra.program.model.util.AddressSetPropertyMap;
 import ghidra.program.model.util.CodeUnitInsertionException;
+import ghidra.program.util.ExternalSymbolResolver;
 import ghidra.util.*;
 import ghidra.util.datastruct.*;
 import ghidra.util.exception.*;
@@ -454,8 +455,7 @@ class ElfProgramBuilder extends MemorySectionResolver implements ElfLoadHelper {
 			String[] neededLibs = elf.getDynamicLibraryNames();
 			for (String neededLib : neededLibs) {
 				monitor.checkCanceled();
-				props.setString(
-					ElfLoader.ELF_REQUIRED_LIBRARY_PROPERTY_PREFIX + pad(libraryIndex++) + "]",
+				props.setString(ExternalSymbolResolver.getRequiredLibraryProperty(libraryIndex++),
 					neededLib);
 			}
 		}

Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/MachoProgramBuilder.java

@@ -48,6 +48,7 @@ import ghidra.program.model.reloc.RelocationResult;
 import ghidra.program.model.reloc.RelocationTable;
 import ghidra.program.model.symbol.*;
 import ghidra.program.model.util.CodeUnitInsertionException;
+import ghidra.program.util.ExternalSymbolResolver;
 import ghidra.util.Msg;
 import ghidra.util.NumericUtilities;
 import ghidra.util.exception.*;
@@ -601,27 +602,46 @@ public class MachoProgramBuilder {
 		}
 	}
 
-	protected void processLibraries() {
+	protected void processLibraries() throws Exception {
 		monitor.setMessage("Processing libraries...");
-		List<LoadCommand> commands = machoHeader.getLoadCommands();
-		for (LoadCommand command : commands) {
+
+		Options props = program.getOptions(Program.PROGRAM_INFO);
+		int libraryIndex = 0;
+
+		for (LoadCommand command : machoHeader.getLoadCommands()) {
 			if (monitor.isCancelled()) {
 				return;
 			}
-			if (command instanceof DynamicLibraryCommand) {
-				DynamicLibraryCommand dylibCommand = (DynamicLibraryCommand) command;
+
+			String libraryPath = null;
+
+			if (command instanceof DynamicLibraryCommand dylibCommand) {
 				DynamicLibrary dylib = dylibCommand.getDynamicLibrary();
-				addLibrary(dylib.getName().getString());
+				libraryPath = dylib.getName().getString();
 			}
-			else if (command instanceof SubLibraryCommand) {
-				SubLibraryCommand sublibCommand = (SubLibraryCommand) command;
-				addLibrary(sublibCommand.getSubLibraryName().getString());
+			else if (command instanceof SubLibraryCommand sublibCommand) {
+				libraryPath = sublibCommand.getSubLibraryName().getString();
 			}
-			else if (command instanceof PreboundDynamicLibraryCommand) {
-				PreboundDynamicLibraryCommand pbdlCommand = (PreboundDynamicLibraryCommand) command;
-				addLibrary(pbdlCommand.getLibraryName());
+			else if (command instanceof PreboundDynamicLibraryCommand pbdlCommand) {
+				libraryPath = pbdlCommand.getLibraryName();
+			}
+
+			if (libraryPath != null) {
+				// For now, strip off the full path and just the use the filename.  We will utilize
+				// the full path one day when we started looking in dyld_shared_cache files for
+				// libraries.
+				int index = libraryPath.lastIndexOf("/");
+				String libraryName = index != -1 ? libraryPath.substring(index + 1) : libraryPath;
+				if (!libraryName.equals(program.getName())) {
+					addLibrary(libraryName);
+					props.setString(
+						ExternalSymbolResolver.getRequiredLibraryProperty(libraryIndex++),
+						libraryName);
+				}
 			}
 		}
+
+		program.getSymbolTable().createExternalLibrary(Library.UNKNOWN, SourceType.IMPORTED);
 	}
 
 	protected void processProgramDescription() {

Ghidra/Features/Base/src/main/java/ghidra/program/util/ExternalSymbolResolver.java

@@ -19,65 +19,83 @@ import java.io.IOException;
 import java.util.*;
 import java.util.stream.Collectors;
 
+import db.Transaction;
 import ghidra.app.util.importer.MessageLog;
-import ghidra.app.util.opinion.ElfLoader;
 import ghidra.app.util.opinion.Loaded;
-import ghidra.framework.model.DomainObject;
 import ghidra.framework.options.Options;
 import ghidra.program.model.listing.*;
 import ghidra.program.model.symbol.*;
 import ghidra.util.Msg;
+import ghidra.util.StringUtilities;
 import ghidra.util.exception.*;
 import ghidra.util.task.TaskMonitor;
 
-public class ELFExternalSymbolResolver {
+public class ExternalSymbolResolver {
+
+	private final static String REQUIRED_LIBRARY_PROPERTY_PREFIX = "Required Library [";
+
+	/**
+	 * Gets a program property name to represent the ordered required library of the given index
+	 * 
+	 * @param libraryIndex The index of the required library
+	 * @return A program property name to represent the ordered required library of the given index
+	 */
+	public static String getRequiredLibraryProperty(int libraryIndex) {
+		return String.format("%s %s]", REQUIRED_LIBRARY_PROPERTY_PREFIX,
+			StringUtilities.pad("" + libraryIndex, ' ', 4));
+	}
 
 	/**
 	 * Links unresolved symbols to the first symbol found in the (ordered) linked
-	 * libraries (saved in the program's properties as "ELF Required Library [nn]").
+	 * libraries (saved in the program's properties as {@value #REQUIRED_LIBRARY_PROPERTY_PREFIX}).
 	 * <p>
-	 * The ordering and precedence logic is ELF specific though no ELF binary formats
+	 * The ordering and precedence logic is loader specific though no particular binary formats
 	 * are parsed or required.
 	 * <p>
 	 * The program's external libraries need to already be populated with paths to
 	 * already existing / imported libraries.
-	 * <p>
 	 *
-	 * @param loadedPrograms ELF {@link Loaded} {@link Program}s to fix..
+	 * @param loadedPrograms The {@link Loaded} {@link Program}s to fix.  The first entry is the
+	 *   "primary" {@link Loaded} {@link Program}.
+	 * @param fixAll True if all of the {@link Loaded} {@link Program}s should be fixed;
+	 *   false if just the "primary" {@link Loaded} {@link Program} should be fixed.
 	 * @param messageLog {@link MessageLog} to write info message to.
 	 * @param monitor {@link TaskMonitor} to watch for cancel and update with progress.
 	 * @throws CancelledException if user cancels
 	 * @throws IOException if error reading
 	 */
 	public static void fixUnresolvedExternalSymbols(List<Loaded<Program>> loadedPrograms,
-			MessageLog messageLog, TaskMonitor monitor) throws CancelledException, IOException {
+			boolean fixAll, MessageLog messageLog, TaskMonitor monitor)
+			throws CancelledException, IOException {
 		Map<String, Loaded<Program>> loadedByName = loadedPrograms.stream()
 				.collect(
 					Collectors.toMap(loaded -> loaded.getName(), loaded -> loaded));
 
-		monitor.initialize(loadedByName.size());
-		for (Loaded<Program> loadedProgram : loadedByName.values()) {
+		List<Loaded<Program>> fixupList =
+			loadedPrograms.subList(0, fixAll ? loadedPrograms.size() : 1);
+
+		monitor.initialize(fixupList.size());
+		for (Loaded<Program> loadedProgram : fixupList) {
 			Program program = loadedProgram.getDomainObject();
 
 			Collection<Long> unresolvedExternalFunctionIds =
 				getUnresolvedExternalFunctionIds(program);
 			if (unresolvedExternalFunctionIds.size() == 0) {
-				return;
+				continue;
 			}
 
 			List<Library> libSearchList = getLibrarySearchList(program);
 			if (libSearchList.isEmpty()) {
-				return;
+				continue;
 			}
 
-			int transactionID = program.startTransaction("Resolve External Symbols");
-			try {
+			try (Transaction tx = program.openTransaction("Resolve External Symbols")) {
 
 				messageLog.appendMsg("----- [" + program.getName() + "] Resolve " +
 					unresolvedExternalFunctionIds.size() + " external symbols -----");
 
 				for (Library extLibrary : libSearchList) {
-					monitor.checkCanceled();
+					monitor.checkCancelled();
 					String libName = extLibrary.getName();
 					String libPath = extLibrary.getAssociatedProgramPath();
 					if (libPath == null) {
@@ -90,17 +108,14 @@ public class ELFExternalSymbolResolver {
 						continue;
 					}
 
-					DomainObject libDomainObject = loadedLib.getDomainObject();
+					Program libProgram = loadedLib.getDomainObject();
 					monitor.setMessage("Resolving symbols published by library " + libName);
 					resolveSymbolsToLibrary(program, unresolvedExternalFunctionIds, extLibrary,
-						(Program) libDomainObject, messageLog, monitor);
+						libProgram, messageLog, monitor);
 				}
 				messageLog.appendMsg("Unresolved external symbols which remain: " +
 					unresolvedExternalFunctionIds.size());
 			}
-			finally {
-				program.endTransaction(transactionID, true);
-			}
 		}
 	}
 
@@ -113,10 +128,10 @@ public class ELFExternalSymbolResolver {
 
 		Iterator<Long> idIterator = unresolvedExternalFunctionIds.iterator();
 		while (idIterator.hasNext()) {
-			monitor.checkCanceled();
+			monitor.checkCancelled();
 			Symbol s = symbolTable.getSymbol(idIterator.next());
 			if (s == null || !s.isExternal() || s.getSymbolType() != SymbolType.FUNCTION) {
-				Msg.error(ELFExternalSymbolResolver.class,
+				Msg.error(ExternalSymbolResolver.class,
 					"Concurrent modification of symbol table while resolving external symbols");
 				idIterator.remove();
 				continue;
@@ -131,11 +146,11 @@ public class ELFExternalSymbolResolver {
 				s.setNamespace(extLibrary);
 				idIterator.remove();
 				libResolvedCount++;
-				Msg.debug(ELFExternalSymbolResolver.class, "External symbol " + extLoc.getLabel() +
+				Msg.debug(ExternalSymbolResolver.class, "External symbol " + extLoc.getLabel() +
 					" resolved to " + extLibrary.getName());
 			}
 			catch (DuplicateNameException | InvalidInputException | CircularDependencyException e) {
-				Msg.error(ELFExternalSymbolResolver.class,
+				Msg.error(ExternalSymbolResolver.class,
 					"Error setting external symbol namespace for " + extLoc.getLabel(), e);
 			}
 		}
@@ -175,24 +190,29 @@ public class ELFExternalSymbolResolver {
 	private static Collection<String> getOrderedLibraryNamesNeeded(Program program) {
 		TreeMap<Integer, String> orderLibraryMap = new TreeMap<>();
 		Options options = program.getOptions(Program.PROGRAM_INFO);
-		//List<String> sortedOptionNames = new ArrayList<>();
 		for (String optionName : options.getOptionNames()) {
-			if (!optionName.startsWith(ElfLoader.ELF_REQUIRED_LIBRARY_PROPERTY_PREFIX) ||
-				!optionName.endsWith("]")) {
+
+			// Legacy programs may have the old "ELF Required Library [" program property, so 
+			// we should not assume that the option name starts exactly with 
+			// REQUIRED_LIBRARY_PROPERTY_PREFIX.  We must deal with a potential substring at the
+			// start of the option name.
+			int prefixIndex = optionName.indexOf(REQUIRED_LIBRARY_PROPERTY_PREFIX);
+			if (prefixIndex == -1 || !optionName.endsWith("]")) {
 				continue;
 			}
 			String libName = options.getString(optionName, null);
 			if (libName == null) {
 				continue;
 			}
-			String indexStr =
-				optionName.substring(ElfLoader.ELF_REQUIRED_LIBRARY_PROPERTY_PREFIX.length(),
-					optionName.length() - 1).trim();
+			String indexStr = optionName
+					.substring(prefixIndex + REQUIRED_LIBRARY_PROPERTY_PREFIX.length(),
+						optionName.length() - 1)
+					.trim();
 			try {
 				orderLibraryMap.put(Integer.parseInt(indexStr), libName.trim());
 			}
 			catch (NumberFormatException e) {
-				Msg.error(ELFExternalSymbolResolver.class,
+				Msg.error(ExternalSymbolResolver.class,
 					"Program contains invalid property: " + optionName);
 			}
 		}

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/symbol/LibraryDB.java

@@ -116,4 +116,9 @@ class LibraryDB implements Library {
 		return true;
 	}
 
+	@Override
+	public String toString() {
+		return getName();
+	}
+
 }