Various updates

script/buildDist.py

@@ -38,15 +38,68 @@ class FancyArgumentParser(argparse.ArgumentParser):
 #			yield arg
 
 
+def buildCatalogFile(deltaPath):
+
+	# Build the delta md5sum catalog
+	records = []
+
+	snipLen = len(deltaPath) + 1
+#	for root, dirNames, fileNames in os.walk(deltaPath, onerror=failTracker.recordError):
+	for root, dirNames, fileNames in os.walk(deltaPath):
+
+		# Presort the results alphabetically
+		dirNames.sort()
+		fileNames.sort()
+
+		# Form the record for the current directory (PathNode)
+		fullPath = root
+		relPath = fullPath[snipLen:]
+		if len(relPath) > 0:
+			record = ('P', relPath)
+			records.append(record)
+
+		# Since we do not visit symbolic links, notify the user of all directories that are symbolic
+		for dirName in dirNames:
+			fullPath = os.path.join(root, dirName)
+			if os.path.islink(fullPath) == True:
+				print("Path links are not supported... Skipping: " + fullPath + "\n")
+
+		# Record all of the file nodes
+		for fileName in fileNames:
+			fullPath = os.path.join(root, fileName)
+			if os.path.islink(fullPath) == True:
+				print("File links are not supported... Skipping: " + fullPath + "\n")
+			elif os.path.isfile(fullPath) == True:
+				# Gather the various stats of the specified file
+				stat = os.stat(fullPath)
+				md5sum = miscUtils.computeMd5ForFile(fullPath)
+				relPath = fullPath[snipLen:]
+				record = ('F', md5sum, str(stat.st_size), relPath)
+				records.append(record)
+			else:
+				print("Undefined node. Full path: " + fullPath + "\n")
+
+
+	# Save the records to the catalog file
+	dstPath = os.path.join(deltaPath, "catalog.txt")
+	f = open(dstPath, 'wb')
+	for aRecord in records:
+		if len(aRecord) == 2:
+			f.write(aRecord[0] + ',' + aRecord[1] + '\n')
+		else:
+			f.write(aRecord[0] + ',' + aRecord[1] + ',' + aRecord[2] + ',' + aRecord[3] + '\n')
+	f.close()
+
+
 def getClassPath(javaCodePath):
 	retList = []
-	
+
 	# Ensure the javaCodePath has a trailing slash
 	# to allow for proper computation of clipLen 
 	if javaCodePath.endswith('/') == False:
 		javaCodePath += '/'
 	clipLen = len(javaCodePath)
-	
+
 	# Form the default list of all jar files
 	for path, dirs, files in os.walk(javaCodePath):
 		for file in files:
@@ -55,7 +108,7 @@ def getClassPath(javaCodePath):
 				filePath = filePath[clipLen:]
 				retList.append(filePath)
 #				print('Found jar file at: ' + filePath)
-				
+
 	return retList
 
 
@@ -65,7 +118,7 @@ if __name__ == "__main__":
 
 	# Logic to capture Ctrl-C and bail
 	signal.signal(signal.SIGINT, miscUtils.handleSignal)
-	
+
 	# Set up the argument parser	
 	parser = FancyArgumentParser(prefix_chars='-', add_help=False, fromfile_prefix_chars='@')
 	parser.add_argument('-help', '-h', help='Show this help message and exit.', action='help')
@@ -77,8 +130,9 @@ if __name__ == "__main__":
 	parser.add_argument('-javaCode', '-jc', help='A folder which contains the Java build.')
 	parser.add_argument('-jvmArgs', help='JVM arguments.', nargs='+', default=[])
 	parser.add_argument('-classPath', help='Class path listing of jar files relative to javaCode. Leave blank for auto determination.', nargs='+', default=[])
+	parser.add_argument('-debug', help='Turn on debug options for built applications.', action='store_true', default=False)
 	parser.add_argument('-company', help='Company / Provider info.')
-	parser.add_argument('-bgFile',  help='Background file used for apple dmg file.')
+	parser.add_argument('-bgFile', help='Background file used for apple dmg file.')
 	parser.add_argument('-iconFile', help='PNG file used for linux/windows icon.')
 	parser.add_argument('-icnsFile', help='Icon file used for apple build.')
 	parser.add_argument('-forceSingleInstance', help='Force the application to have only one instance..', default=False)
@@ -91,24 +145,24 @@ if __name__ == "__main__":
 			exit()
 
 	# Parse the args		
-	parser.formatter_class.max_help_position = 50	
+	parser.formatter_class.max_help_position = 50
 	args = parser.parse_args()
 #	print args
-	
+
 	# Ensure we are getting the bare minimum options
 	if args.name == None:
 		print('At a minimum the application name must be specified. Exiting...')
 		exit();
-		
+
 	# Ensure java options are specified properly
 	if (args.javaCode == None and args.mainClass != None) or (args.javaCode != None and args.mainClass == None):
 		print('Both javaCode and mainClass must be specified, if either are specified. Exiting...')
 		exit();
-	
+
 	# Form the classPath if none specified
 	if args.javaCode != None and len(args.classPath) == 0:
 		args.classPath = getClassPath(args.javaCode)
-		
+
 	# Clean up the jvmArgs to replace the escape sequence '\-' to '-'
 	# and to ensure that all the args start with the '-' character
 	newJvmArgs = []
@@ -122,62 +176,52 @@ if __name__ == "__main__":
 			newJvmArgs.append(aJvmArg)
 	args.jvmArgs = newJvmArgs
 
-		
+
 	# Bail if the release has already been built
 	buildPath = os.path.abspath(args.name + '-' + args.version)
 	if (os.path.exists(buildPath) == True):
 		print('   [ERROR] The release appears to be built. Path: ' + buildPath)
 		exit(-1)
-	
+
 	# Form the buildPath
 	os.makedirs(buildPath)
-	
+
 	# Build the Delta (diffs) contents
-	deltaCodePath = os.path.join(buildPath, "delta/code")
-	deltaDataPath = os.path.join(buildPath, "delta/data")
-	
+	deltaPath = os.path.join(buildPath, "delta")
+	deltaCodePath = os.path.join(deltaPath, "code")
+	deltaDataPath = os.path.join(deltaPath, "data")
+
 	# Copy the dataCode to the delta location
 	os.makedirs(deltaDataPath)
 	for aPath in args.dataCode:
 		srcPath = aPath
 		dstPath = os.path.join(deltaDataPath, os.path.basename(aPath))
-		shutil.copytree(srcPath, dstPath, symlinks=True)
-		
+		shutil.copytree(srcPath, dstPath, symlinks=False)
+
 	# Build the java component of the distribution
 	if args.javaCode != None:
 		# Copy the javaCode to the proper location
 		srcPath = args.javaCode
 		dstPath = deltaCodePath;
-#		dstPath = os.path.join(deltaCodePath, 'java')
-		shutil.copytree(srcPath, dstPath, symlinks=True)
-		
+		shutil.copytree(srcPath, dstPath, symlinks=False)
+
 	# Form the app.cfg file	
 	dstPath = os.path.join(buildPath, "delta/app.cfg")
 	miscUtils.buildAppLauncherConfig(dstPath, args)
-	
+
 	# Build the delta md5sum catalog
-	# Use extreme caution when editing the line below since we have enabled the shell!!!
-	# Currently there is no external (uncleansed) input in the command, and be sure to not introduce it!
-#	dstPath = os.path.join(buildPath, "delta/md5sum.txt")
-#	buildCatalog.buildCatalog(destPath)
-	dstPath = os.path.join(buildPath, "delta")
-	subprocess.check_call("find * -name '*' -type f -print0 | sort -d -z | xargs -0 md5sum > ../md5sum.txt", cwd=dstPath, shell=True)
-	
-	# Move the md5sum catalog into the delta folder
-	srcPath = os.path.join(buildPath, "md5sum.txt")
-	dstPath = os.path.join(buildPath, "delta/md5sum.txt")
-	os.rename(srcPath, dstPath)
-		
+	buildCatalogFile(deltaPath)
+
 	# Build the Apple release	
 	appleUtils.buildRelease(args, buildPath)
-	
+
 	# Build the Linux release
 	linuxUtils.buildRelease(args, buildPath)
-	
+
 	# Build the Windows release
 	windowsUtils.buildRelease(args, buildPath)
-	
+
 	# Copy over the deploy script
 	srcPath = os.path.join(miscUtils.getInstallRoot(), "deployDist.py")
 	shutil.copy(srcPath, buildPath)
-	
+

script/deployDist.py

@@ -99,7 +99,7 @@ if __name__ == "__main__":
 	# Set up the argument parser	
 	parser = argparse.ArgumentParser(prefix_chars='-', add_help=False, fromfile_prefix_chars='@')
 	parser.add_argument('-help', '-h', help='Show this help message and exit.', action='help')
-	parser.add_argument('rootLoc', help='Root location to deploy the specified distribution.')
+	parser.add_argument('deployRoot', help='Root location to deploy the specified distribution.')
 	parser.add_argument('distLoc', nargs='?', default=scriptPath, help='The location of the distribution to deploy.')
 
 	# Intercept any request for a  help message and bail
@@ -113,7 +113,7 @@ if __name__ == "__main__":
 	args = parser.parse_args()
 	
 	distPath = args.distLoc
-	rootPath = args.rootLoc
+	rootPath = args.deployRoot
 
 	# Retrieve the distPath and ensure that it exists 
 	if os.path.isdir(distPath) == False:
@@ -127,7 +127,7 @@ if __name__ == "__main__":
 	# Check to see if the deployed location already exists
 	installPath = os.path.join(rootPath, appName)
 	if os.path.isdir(installPath) == False:
-		print('Application ' + appName + ' has never been deployed to the root location: ' + args.rootLoc)
+		print('Application ' + appName + ' has never been deployed to the root location: ' + args.deployRoot)
 		print('Create a new release of the application at the specified location?')
 		input = raw_input('--> ').upper()
 		if input != 'Y' and input != 'YES':

script/miscUtils.py

@@ -1,4 +1,5 @@
 #! /usr/bin/env python
+import hashlib
 import os
 import time
 import sys
@@ -112,3 +113,16 @@ def buildAppLauncherConfig(destFile, args):
 	f.write('\n')
 
 	f.close()
+
+
+# Source: http://stackoverflow.com/questions/1131220/get-md5-hash-of-a-files-without-open-it-in-python
+def computeMd5ForFile(evalFile, block_size=2**20):
+	f = open(evalFile, 'rb')
+	md5 = hashlib.md5()
+	while True:
+		data = f.read(block_size)
+		if not data:
+			break
+		md5.update(data)
+	f.close()
+	return md5.hexdigest()

script/windowsUtils.py

@@ -107,8 +107,10 @@ def buildLaunch4JConfig(destFile, args, isStaticRelease, iconFile):
 	f = open(destFile, 'wb')
 
 	writeln(f, 0, "<launch4jConfig>")
-	writeln(f, 1, "<headerType>gui</headerType>");
-#	writeln(f, 1, "<headerType>console</headerType>");
+	if args.debug == True:
+		writeln(f, 1, "<headerType>console</headerType>");
+	else:
+		writeln(f, 1, "<headerType>gui</headerType>");
 	writeln(f, 1, "<outfile>" + args.name + ".exe</outfile>");
 	writeln(f, 1, "<dontWrapJar>true</dontWrapJar>");
 	writeln(f, 1, "<errTitle>" + args.name + "</errTitle>");

src/distMaker/DistMakerEngine.java

@@ -11,18 +11,21 @@ import glum.unit.DateUnit;
 import glum.util.ThreadUtil;
 
 import java.io.*;
+import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.List;
+import java.util.Map;
 
 import javax.swing.JFrame;
 import javax.swing.SwingUtilities;
 
 import distMaker.gui.PickReleasePanel;
+import distMaker.node.Node;
 
 public class DistMakerEngine
 {
 	// State vars
-	private URL updateUrl;
+	private URL updateSiteUrl;
 	private Release currRelease;
 	private Credential refCredential;
 
@@ -32,9 +35,9 @@ public class DistMakerEngine
 	private PromptPanel promptPanel;
 	private PickReleasePanel pickVersionPanel;
 
-	public DistMakerEngine(JFrame aParentFrame, URL aUpdateUrl)
+	public DistMakerEngine(JFrame aParentFrame, URL aUpdateSiteUrl)
 	{
-		updateUrl = aUpdateUrl;
+		updateSiteUrl = aUpdateSiteUrl;
 		currRelease = null;
 		refCredential = null;
 
@@ -206,13 +209,13 @@ public class DistMakerEngine
 	{
 		final List<Release> fullList;
 		Release chosenItem;
-		final File installPath, destPath;
+		final File installPath, deltaPath;
 		String appName;
 		boolean isPass;
 
 		// Determine the path to download updates
 		installPath = DistUtils.getAppPath().getParentFile();
-		destPath = new File(installPath, "delta");
+		deltaPath = new File(installPath, "delta");
 
 		// Status info
 		appName = currRelease.getName();
@@ -220,7 +223,7 @@ public class DistMakerEngine
 
 		// Retrieve the list of available releases
 		aTask.infoAppendln("Checking for updates...\n");
-		fullList = DistUtils.getAvailableReleases(aTask, updateUrl, appName, refCredential);
+		fullList = DistUtils.getAvailableReleases(aTask, updateSiteUrl, appName, refCredential);
 		if (fullList == null)
 		{
 			aTask.abort();
@@ -237,7 +240,7 @@ public class DistMakerEngine
 				public void run()
 				{
 					// Query the user, if the wish to destroy the old update
-					if (destPath.isDirectory() == true)
+					if (deltaPath.isDirectory() == true)
 					{
 						promptPanel.setTitle("Overwrite recent update?");
 						promptPanel.setInfo("An update has already been downloaded... If you proceed this update will be removed. Proceed?");
@@ -248,7 +251,7 @@ public class DistMakerEngine
 							return;
 						}
 
-						IoUtil.deleteDirectory(destPath);
+						IoUtil.deleteDirectory(deltaPath);
 					}
 
 					// Query the user of the version to update to
@@ -277,11 +280,21 @@ public class DistMakerEngine
 		else
 			aTask.infoAppendln("\t" + appName + " will be reverted...");
 
-		// Download the release
-		isPass = downloadRelease(aTask, chosenItem, destPath);
+		// Form the destination path
+		isPass = deltaPath.mkdirs();
 		if (isPass == false || aTask.isActive() == false)
 		{
-			IoUtil.deleteDirectory(destPath);
+			aTask.infoAppendln("Failed to create delta path: " + deltaPath);
+			aTask.infoAppendln("Application update aborted.");
+			aTask.abort();
+			return;
+		}
+
+		// Download the release
+		isPass = downloadRelease(aTask, chosenItem, deltaPath);
+		if (isPass == false || aTask.isActive() == false)
+		{
+			IoUtil.deleteDirectory(deltaPath);
 			aTask.infoAppendln("Application update aborted.");
 			aTask.abort();
 			return;
@@ -300,42 +313,68 @@ public class DistMakerEngine
 	 */
 	private boolean downloadRelease(Task aTask, Release aRelease, File destPath)
 	{
-		List<File> fileList;
-		String baseUrlStr, srcUrlStr;
-		URL srcUrl;
-		int clipLen;
+		Map<String, Node> currMap, updateMap;
+		URL staleUrl, updateUrl;
+		Node staleNode, updateNode;
 		boolean isPass;
 
-		// Retrieve the list of files to download
-		fileList = DistUtils.getFileListForRelease(aTask, updateUrl, aRelease, destPath, refCredential);
-		if (fileList == null)
+		try
+		{
+			staleUrl = DistUtils.getAppPath().toURI().toURL();
+			updateUrl = IoUtil.createURL(updateSiteUrl.toString() + "/" + aRelease.getName() + "/" + aRelease.getVersion() + "/delta");
+		}
+		catch (MalformedURLException aExp)
+		{
+			aTask.infoAppendln(ThreadUtil.getStackTrace(aExp));
+			aExp.printStackTrace();
+			return false;
+		}
+
+		// Load the map of stale nodes
+		currMap = DistUtils.readCatalog(aTask, staleUrl, null);
+		if (currMap == null)
 			return false;
 
-		// Compute some baseline vars
-		baseUrlStr = updateUrl.toString() + "/" + aRelease.getName() + "/" + aRelease.getVersion() + "/" + "delta/";
-		clipLen = destPath.getAbsolutePath().length() + 1;
+		// Load the map of update nodes
+		updateMap = DistUtils.readCatalog(aTask, updateUrl, refCredential);
+		if (updateMap == null)
+			return false;
 
 		// Download the individual files
-		aTask.infoAppendln("Downloading release: " + aRelease.getVersion() + " Files: " + fileList.size());
-		for (File aFile : fileList)
+		aTask.infoAppendln("Downloading release: " + aRelease.getVersion() + " Nodes: " + updateMap.size());
+		for (String aFileName : updateMap.keySet())
 		{
 			// Bail if we have been aborted
 			if (aTask.isActive() == false)
 				return false;
 
-			srcUrlStr = baseUrlStr + aFile.getAbsolutePath().substring(clipLen);
-			srcUrl = IoUtil.createURL(srcUrlStr);
+			updateNode = updateMap.get(aFileName);
+			staleNode = currMap.get(aFileName);
 
-//			aTask.infoAppendln("\t" + srcUrlStr + " -> " + aFile);
-			aTask.infoAppendln("\t(U) " + aFile);
-			aFile.getParentFile().mkdirs();
-			isPass = DistUtils.downloadFile(aTask, srcUrl, aFile, refCredential);
+			// Attempt to use the local copy
+			isPass = false;
+			if (staleNode != null && updateNode.areContentsEqual(staleNode) == true)
+			{
+				isPass = staleNode.transferContentTo(aTask, refCredential, destPath);
+				if (isPass == true)
+					aTask.infoAppendln("\t(L) " + staleNode.getFileName());
+			}
+
+			// Use the remote update copy
 			if (isPass == false)
 			{
-				aFile.delete();
-				aTask.infoAppendln("Failed to download resource: " + srcUrl);
-				aTask.infoAppendln("\tSource: " + srcUrlStr);
-				aTask.infoAppendln("\tDest: " + aFile);
+				isPass = updateNode.transferContentTo(aTask, refCredential, destPath);
+				if (isPass == true)
+					aTask.infoAppendln("\t(R) " + updateNode.getFileName());
+			}
+
+			// Log the failure and bail
+			if (isPass == false)
+			{
+				aTask.infoAppendln("Failed to download from update site.");
+				aTask.infoAppendln("\tSite: " + updateUrl);
+				aTask.infoAppendln("\tFile: " + updateNode.getFileName());
+				aTask.infoAppendln("\tDest: " + destPath);
 				return false;
 			}
 		}

src/distMaker/DistUtils.java

@@ -1,5 +1,6 @@
 package distMaker;
 
+import glum.gui.GuiUtil;
 import glum.io.IoUtil;
 import glum.net.*;
 import glum.reflect.ReflectUtil;
@@ -10,8 +11,12 @@ import glum.unit.DateUnit;
 import java.io.*;
 import java.net.*;
 import java.util.List;
+import java.util.Map;
 
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import distMaker.node.*;
 
 public class DistUtils
 {
@@ -228,9 +233,9 @@ public class DistUtils
 	/**
 	 * Returns the list of files (relative to destPath) that are needed for the specified release.
 	 */
-	public static List<File> getFileListForRelease(Task aTask, URL aUpdateUrl, Release aRelease, File destPath, Credential aCredential)
+	public static Map<String, Node> readCatalog(Task aTask, URL aUpdateUrl, Credential aCredential)
 	{
-		List<File> fullList;
+		Map<String, Node> retMap;
 		URL md5sumUrl;
 		URLConnection connection;
 		InputStream inStream;
@@ -238,8 +243,8 @@ public class DistUtils
 		String errMsg;
 
 		errMsg = null;
-		fullList = Lists.newArrayList();
-		md5sumUrl = IoUtil.createURL(aUpdateUrl.toString() + "/" + aRelease.getName() + "/" + aRelease.getVersion() + "/delta/md5sum.txt");
+		retMap = Maps.newLinkedHashMap();
+		md5sumUrl = IoUtil.createURL(aUpdateUrl.toString() + "/catalog.txt");
 
 		connection = null;
 		inStream = null;
@@ -247,7 +252,8 @@ public class DistUtils
 		try
 		{
 			String[] tokens;
-			String strLine, filename;
+			String strLine, filename, md5sum;
+			long fileLen;
 
 			// Read the contents of the file
 			connection = md5sumUrl.openConnection();
@@ -263,16 +269,24 @@ public class DistUtils
 				if (strLine == null)
 					break;
 
-				tokens = strLine.split("\\s+");
+				tokens = strLine.split(",", 4);
 				if (strLine.isEmpty() == true || strLine.startsWith("#") == true)
 					; // Nothing to do
-				else if (tokens.length != 2)
-					aTask.infoAppendln("Unreconized line: " + strLine);
-				else
-				// if (tokens.length == 2)
+				else if (tokens.length == 2 && tokens[0].equals("P") == true)
 				{
 					filename = tokens[1];
-					fullList.add(new File(destPath, filename));
+					retMap.put(filename, new PathNode(aUpdateUrl, filename));
+				}
+				else if (tokens.length == 4 && tokens[0].equals("F") == true)
+				{
+					md5sum = tokens[1];
+					fileLen = GuiUtil.readLong(tokens[2], -1);
+					filename = tokens[3];
+					retMap.put(filename, new FileNode(aUpdateUrl, filename, md5sum, fileLen));
+				}
+				else
+				{
+					aTask.infoAppendln("Unreconized line: " + strLine);
 				}
 			}
 		}
@@ -289,7 +303,7 @@ public class DistUtils
 		// See if we are in a valid state
 		if (errMsg != null)
 			; // Nothing to do, as an earlier error has occured
-		else if (fullList.size() == 0)
+		else if (retMap.size() == 0)
 			errMsg = "The md5sum URL appears to be invalid.";
 
 		// Bail if there were issues
@@ -299,7 +313,7 @@ public class DistUtils
 			return null;
 		}
 
-		return fullList;
+		return retMap;
 	}
 
 	/**