From 3a286b7087968bf1645c2896cc2a43fa44fab9b0 Mon Sep 17 00:00:00 2001 From: Norberto Lopez <106690103+nobes888@users.noreply.github.com> Date: Tue, 8 Mar 2016 20:28:40 +0000 Subject: [PATCH] Code cleanup. Fixed application argument processing. Fixed memory configuration logic under Linux. Improved scripts to make them more robust. --- buildDistMakerBin.jardesc | 2 +- script/appleUtils.py | 49 ++++++---- script/buildDist.py | 72 +++++++++++++-- script/jreUtils.py | 86 ++++++++++++++++++ script/linuxUtils.py | 77 +++++++++++----- script/logUtils.py | 24 +++++ script/miscUtils.py | 110 ++++++++++++++--------- script/windowsUtils.py | 63 +++++++++---- src/distMaker/DistMakerEngine.java | 70 +++++++++------ src/distMaker/DistUtils.java | 77 +++++++++------- src/distMaker/LoggingTask.java | 9 +- src/distMaker/gui/MemoryConfigPanel.java | 3 + src/distMaker/gui/PickReleasePanel.java | 25 ++---- src/distMaker/node/FileNode.java | 12 ++- src/distMaker/platform/LinuxUtils.java | 104 +++++++++++++++------ src/distMaker/platform/MemUtils.java | 26 ++++-- src/distMaker/platform/WindowsUtils.java | 23 +++-- src/dsstore/AliasRecord.java | 7 +- src/dsstore/BlockDir.java | 15 ++-- src/dsstore/MainApp.java | 68 +++++++------- 20 files changed, 640 insertions(+), 282 deletions(-) create mode 100644 script/jreUtils.py create mode 100644 script/logUtils.py diff --git a/buildDistMakerBin.jardesc b/buildDistMakerBin.jardesc index 8709eb1..7d86626 100644 --- a/buildDistMakerBin.jardesc +++ b/buildDistMakerBin.jardesc @@ -1,4 +1,4 @@ - + diff --git a/script/appleUtils.py b/script/appleUtils.py index f1ab645..71bcf37 100644 --- a/script/appleUtils.py +++ b/script/appleUtils.py @@ -1,6 +1,7 @@ #! /usr/bin/env python import argparse +import copy import getpass import math import os @@ -12,28 +13,39 @@ import tempfile import time import glob +import jreUtils import miscUtils -# Globals -# The global var jre is a pointer to a full jre release for an Apple system -# There should be a release located at /jre/apple/jreRelease -jreRelease='jre1.7.0_51' - - def buildRelease(args, buildPath): + # We may mutate args - thus make a custom copy + args = copy.copy(args) + # Retrieve vars of interest appName = args.name version = args.version + jreRelease = args.jreRelease + # Attempt to locate a default JRE if none is specified in the args. + if jreRelease == None: + jreRelease = jreUtils.getDefaultJreRelease('apple') + # Let the user know if the 'user' specified JRE is not available and locate an alternative + elif jreUtils.isJreAvailable('apple', jreRelease) == False: + print('[Warning] User specified JRE (' + jreRelease + ') is not available for Apple platform. Searching for alternative...') + jreRelease = jreUtils.getDefaultJreRelease('apple') + args.jreRelease = jreRelease + # Form the list of distributions to build distList = [(appName + '-' + version, False)] - if miscUtils.isJreAvailable('apple', jreRelease) == True: + if jreUtils.isJreAvailable('apple', jreRelease) == True: distList.append((appName + '-' + version + '-jre', True)) # Create the various Apple distributions for (distName, isStaticRelease) in distList: print('Building Apple distribution: ' + distName) + # Let the user know of the JRE release we are going to build with + if isStaticRelease == True: + print('\tUtilizing jreRelease: ' + jreRelease) # Create a tmp folder and build the static release to the tmp folder tmpPath = tempfile.mkdtemp(dir=buildPath) @@ -45,10 +57,14 @@ def buildRelease(args, buildPath): dmgFile = os.path.join(buildPath, distName + '.dmg') # cmd = ['genisoimage', '-o', dmgFile, '-quiet', '-V', appName, '-max-iso9660-filenames', '-hfs-unlock', '-uid', '501', '-gid', '80', '-r', '-D', '-apple', tmpPath] cmd = ['genisoimage', '-o', dmgFile, '-quiet', '-V', appName, '-max-iso9660-filenames', '-hfs-unlock', '-D', '-r', '-apple', tmpPath] - subprocess.call(cmd, stderr=subprocess.STDOUT) - + print('\tForming DMG image. File: ' + dmgFile) + proc = miscUtils.executeAndLog(cmd, "\t\tgenisoimage: ") + if proc.returncode != 0: + print('\tError: Failed to form DMG image. Return code: ' + str(proc.returncode)) + # Perform cleanup: Remove the tmp folder shutil.rmtree(tmpPath) + print('\tFinished building release: ' + os.path.basename(dmgFile)) @@ -117,6 +133,7 @@ def buildDistTree(buildPath, rootPath, args, isStaticRelease): appName = args.name bgFile = args.bgFile icnsFile = args.icnsFile + jreRelease = args.jreRelease # Form the symbolic link which points to /Applications srcPath = '/Applications' @@ -147,7 +164,6 @@ def buildDistTree(buildPath, rootPath, args, isStaticRelease): dstPath = os.path.join(rootPath, appName + '.app', 'Contents', 'MacOS') shutil.copy(srcPath, dstPath) - # Write out the PkgInfo file dstPath = os.path.join(rootPath, appName + '.app', 'Contents', "PkgInfo") f = open(dstPath, 'wb') @@ -212,15 +228,18 @@ def buildDistTree(buildPath, rootPath, args, isStaticRelease): shutil.copy(srcPath, dstPath) # Copy over the .DS_Store - srcPath = os.path.join(appTemplatePath, '.DS_Store') - dstPath = rootPath + srcPath = os.path.join(appTemplatePath, '.DS_Store.template') + dstPath = os.path.join(rootPath, '.DS_Store') shutil.copy(srcPath, dstPath) # Update the .DS_Store file to reflect the new volume name srcPath = os.path.join(rootPath, '.DS_Store') - classPath = appInstallRoot + '/lib/glum.jar:' + appInstallRoot + '/lib/distMaker.jar:' + appInstallRoot + '/lib/guava-13.0.1.jar' + classPath = appInstallRoot + '/lib/glum.jar:' + appInstallRoot + '/lib/distMaker.jar:' + appInstallRoot + '/lib/guava-18.0.jar' cmd = ['java', '-cp', classPath, 'dsstore.MainApp', srcPath, appName] - subprocess.check_call(cmd, stderr=None, stdout=None) + proc = miscUtils.executeAndLog(cmd, "\t\tdsstore.MainApp: ") + if proc.returncode != 0: + print('\tError: Failed to update .DS_Store. Return code: ' + str(proc.returncode)) + @@ -368,7 +387,7 @@ def buildPListInfoStatic(destFile, args): tupList.append(('CFBundleSignature', '????')) tupList.append(('CFBundleVersion', args.version)) tupList.append(('NSHumanReadableCopyright', '')) - tupList.append(('JVMRuntime', jreRelease)) + tupList.append(('JVMRuntime', args.jreRelease)) tupList.append(('JVMMainClassName', 'appLauncher.AppLauncher')) diff --git a/script/buildDist.py b/script/buildDist.py index eb45beb..5be75d8 100755 --- a/script/buildDist.py +++ b/script/buildDist.py @@ -1,6 +1,7 @@ #! /usr/bin/env python import argparse +import distutils.spawn import getpass import math import os @@ -90,6 +91,42 @@ def buildCatalogFile(deltaPath): f.write(aRecord[0] + ',' + aRecord[1] + ',' + aRecord[2] + ',' + aRecord[3] + '\n') f.close() +def checkForRequiredApplications(): + """Method to ensure we have all of the required applications installed to support building of distributions + The current set of applications required are: + java, jar, genisoimage, ImageMagick:convert""" + evalPath = distutils.spawn.find_executable('java') + errList = [] + warnList = [] + if evalPath == None: + errList.append('Failed while trying to locate java. Please install Java') + evalPath = distutils.spawn.find_executable('jar') + if evalPath == None: + errList.append('Failed while trying to locate jar. Please install jar (typically included with Java)') + evalPath = distutils.spawn.find_executable('genisoimage') + if evalPath == None: + errList.append('Failed while trying to locate genisoimage. Please install genisoimage') + evalPath = distutils.spawn.find_executable('convert') + if evalPath == None: + warnList.append('Application \'convert\' was not found. Please install (ImageMagick) convert') + warnList.append('\tWindows icons will not be supported when using argument: -iconFile.') + if len(errList) > 0: + print('There are configuration errors with the environment or system.') +# print('System Path:' + str(sys.path)) + print('Please correct the following:') + for aError in errList: + print('\t' + aError) + if len(warnList) > 0: + print('Please correct the following for full program functionality:') + for aWarn in warnList: + print('\t' + aWarn) + sys.exit(0) + if len(warnList) > 0: + print('There are configuration issues with the environment or system.') + print('Please correct the following for full program functionality:') + for aWarn in warnList: + print('\t' + aWarn) + def getClassPath(javaCodePath): retList = [] @@ -102,6 +139,7 @@ def getClassPath(javaCodePath): # Form the default list of all jar files for path, dirs, files in os.walk(javaCodePath): + files.sort() for file in files: if len(file) > 4 and file[-4:] == '.jar': filePath = os.path.join(path, file) @@ -125,9 +163,10 @@ if __name__ == "__main__": parser.add_argument('-name', help='The name of the application.') parser.add_argument('-version', default='0.0.1', help='The version of the application.') parser.add_argument('-mainClass', help='Application main entry point.') - parser.add_argument('-appArgs', help='Application arguments.', nargs='+', default=[]) + parser.add_argument('-appArgs', help='Application arguments. Note that this argument must ALWAYS be the last specified!', nargs=argparse.REMAINDER, default=[]) parser.add_argument('-dataCode', '-dc', help='A list of supporting folders for the application.', nargs='+', default=[]) parser.add_argument('-javaCode', '-jc', help='A folder which contains the Java build.') + parser.add_argument('-jreRelease', help='JRE release to utilize. Note there should be a corresponding folder in ~/jre//', default=None) parser.add_argument('-jvmArgs', help='JVM arguments.', nargs='+', default=[]) parser.add_argument('-classPath', help='Class path listing of jar files relative to javaCode. Leave blank for auto determination.', nargs='+', default=[]) parser.add_argument('-debug', help='Turn on debug options for built applications.', action='store_true', default=False) @@ -144,20 +183,30 @@ if __name__ == "__main__": parser.print_help() exit() + # Check to ensure all of the required applications are installed + checkForRequiredApplications() + # Parse the args parser.formatter_class.max_help_position = 50 args = parser.parse_args() # print args # Ensure we are getting the bare minimum options + errList = []; if args.name == None: - print('At a minimum the application name must be specified. Exiting...') - exit(); - - # Ensure java options are specified properly - if (args.javaCode == None and args.mainClass != None) or (args.javaCode != None and args.mainClass == None): - print('Both javaCode and mainClass must be specified, if either are specified. Exiting...') + errList.append('-name') + if args.javaCode == None: + errList.append('-javaCode') + if args.mainClass == None: + errList.append('-mainClass') + if len(errList) != 0: + print('At a minimum the following must be specified: ' + str(errList) + '.\nExiting...') exit(); +# +# # Ensure java options are specified properly +# if (args.javaCode == None and args.mainClass != None) or (args.javaCode != None and args.mainClass == None): +# print('Both javaCode and mainClass must be specified, if either are specified. Exiting...') +# exit(); # Form the classPath if none specified if args.javaCode != None and len(args.classPath) == 0: @@ -176,7 +225,6 @@ if __name__ == "__main__": newJvmArgs.append(aJvmArg) args.jvmArgs = newJvmArgs - # Bail if the release has already been built buildPath = os.path.abspath(args.name + '-' + args.version) if (os.path.exists(buildPath) == True): @@ -195,6 +243,10 @@ if __name__ == "__main__": os.makedirs(deltaDataPath) for aPath in args.dataCode: srcPath = aPath + if os.path.isdir(srcPath) == False: + print(' [ERROR] The dataCode path does not exist. Path: ' + srcPath) + shutil.rmtree(buildPath) + exit(-1) dstPath = os.path.join(deltaDataPath, os.path.basename(aPath)) shutil.copytree(srcPath, dstPath, symlinks=False) @@ -202,6 +254,10 @@ if __name__ == "__main__": if args.javaCode != None: # Copy the javaCode to the proper location srcPath = args.javaCode + if os.path.isdir(srcPath) == False: + print(' [ERROR] The javaCode path does not exist. Path: ' + srcPath) + shutil.rmtree(buildPath) + exit(-1) dstPath = deltaCodePath; shutil.copytree(srcPath, dstPath, symlinks=False) diff --git a/script/jreUtils.py b/script/jreUtils.py new file mode 100644 index 0000000..115fad2 --- /dev/null +++ b/script/jreUtils.py @@ -0,0 +1,86 @@ +#! /usr/bin/env python + +import os +import re + +import miscUtils + +# Globals +# The global variable defaultJreRelease is a hint for which (full) JRE release to +# default to. Note if this variable is not specified then the latest JRE release +# located in the appropriate platform directory will be utilized. +# +# There should be a release for the following: +# Apple: /jre/apple/defaultJreRelease +# Linux: /jre/linux/defaultJreRelease +# Windows: /jre/windows/defaultJreRelease +defaultJreRelease = 'jre1.8.0_60' + + +def getDefaultJreRelease(aPlatform): + """Utility method to locate the default JRE to utilize""" + # Retrieve vars of interest + appInstallRoot = miscUtils.getInstallRoot() + appInstallRoot = os.path.dirname(appInstallRoot) + + # Locate the JRE platform search path + jreSearchPath = os.path.join(appInstallRoot, 'jre/' + aPlatform) + if os.path.isdir(jreSearchPath) == False: + print ('JRE search path is: ' + jreSearchPath + ' This path does not exist. No embedded JRE builds will be made for platform: ' + aPlatform) + return None + + # Return the default if it is hard wired (and is available) + if 'defaultJreRelease' in globals(): + evalPath = os.path.join(jreSearchPath, defaultJreRelease) + if os.path.isdir(evalPath) == True: + return defaultJreRelease + else: + print('[WARNING] The specified default JRE for the ' + aPlatform.capitalize() + ' platform is: ' + defaultJreRelease + '. However this JRE is not installed! <---') + + # Assume no JRE found + pickJre = None + pickVer = None + + # Search the platform path for available JREs + for aPath in os.listdir(jreSearchPath): + evalPath = os.path.join(jreSearchPath, aPath) + if os.path.isdir(evalPath) == True and aPath.startswith('jre') == True: + tmpVer = re.split('[._]', aPath[3:]) + # print('\tEvaluating path: ' + aPath + ' Ver: ' + str(tmpVer)) + + if pickJre == None: + pickJre = aPath + pickVer = tmpVer + continue + + # Evaluate the version and select the most recent + for i, j in zip(tmpVer, pickVer): + if i < j: + break + if i > j: + pickJre = aPath + pickVer = tmpVer + break + + # Return the result (default JRE) + return pickJre + + +def getJreMajorVersion(aJreRelease): + """Returns the minimum version of the JRE to utilize based on the passed in JRE release. + The passed in value should be of the form jreX.X.X_XX ---> example: jre1.8.0_60 + If aJreRelease is invalid then returns the default value of: 1.8.0""" + if aJreRelease == None or aJreRelease.startswith('jre') == False: + return "1.8.0" + return aJreRelease[3:].split("_")[0] + + +def isJreAvailable(aPlatform, aJreRelease): + """Tests to see if the specified JRE is available for aPlatform""" + if aPlatform == None or aJreRelease == None: + return False + appInstallRoot = miscUtils.getInstallRoot() + appInstallRoot = os.path.dirname(appInstallRoot) + srcPath = os.path.join(appInstallRoot, 'jre', aPlatform, aJreRelease) + return os.path.isdir(srcPath) + diff --git a/script/linuxUtils.py b/script/linuxUtils.py index 677020b..c21d05e 100644 --- a/script/linuxUtils.py +++ b/script/linuxUtils.py @@ -1,6 +1,7 @@ #! /usr/bin/env python import argparse +import copy import getpass import math import os @@ -12,31 +13,42 @@ import tempfile import time import glob +import jreUtils import miscUtils -# Globals -# The global var jre is a pointer to a full jre release for a Linux system -# There should be a release located at /jre/linux/jreRelease -jreRelease = 'jre1.7.0_15' - - def buildRelease(args, buildPath): + # We mutate args - thus make a custom copy + args = copy.copy(args) + # Retrieve vars of interest appName = args.name version = args.version + jreRelease = args.jreRelease + + # Attempt to locate a default JRE if none is specified in the args. + if jreRelease == None: + jreRelease = jreUtils.getDefaultJreRelease('linux') + # Let the user know if the 'user' specified JRE is not available and locate an alternative + elif jreUtils.isJreAvailable('linux', jreRelease) == False: + print('[Warning] User specified JRE (' + jreRelease + ') is not available for Linux platform. Searching for alternative...') + jreRelease = jreUtils.getDefaultJreRelease('linux') + args.jreRelease = jreRelease # Create a tmp (working) folder tmpPath = tempfile.mkdtemp(dir=buildPath) - + # Form the list of distributions to build distList = [(appName + '-' + version, False)] - if miscUtils.isJreAvailable('linux', jreRelease) == True: + if jreUtils.isJreAvailable('linux', jreRelease) == True: distList.append((appName + '-' + version + '-jre', True)) - # Create the various Linux distributions + # Create the various Linux distributions for (distName, isStaticRelease) in distList: print('Building Linux distribution: ' + distName) + # Let the user know of the JRE release we are going to build with + if isStaticRelease == True: + print('\tUtilizing jreRelease: ' + jreRelease) # Create the (top level) distribution folder dstPath = os.path.join(tmpPath, distName) @@ -47,10 +59,12 @@ def buildRelease(args, buildPath): # Create the tar.gz archive tarFile = os.path.join(buildPath, distName + '.tar.gz') + print('\tForming tar.gz file: ' + tarFile) childPath = distName - # print '[' + getCurrTimeStr() + '] Building tar archive: ' + tarFile +# print '[' + getCurrTimeStr() + '] Building tar archive: ' + tarFile subprocess.check_call(["tar", "-czf", tarFile, "-C", tmpPath, childPath], stderr=subprocess.STDOUT) # subprocess.check_call(["gzip", tarFile], stderr=subprocess.STDOUT) + print('\tFinished building release: ' + os.path.basename(tarFile)) # Perform cleanup shutil.rmtree(tmpPath) @@ -61,19 +75,20 @@ def buildDistTree(buildPath, rootPath, args, isStaticRelease): appInstallRoot = miscUtils.getInstallRoot() appInstallRoot = os.path.dirname(appInstallRoot) appName = args.name + jreRelease = args.jreRelease # Form the app contents folder srcPath = os.path.join(buildPath, "delta") dstPath = os.path.join(rootPath, "app") shutil.copytree(srcPath, dstPath, symlinks=True) - #Copy libs to the app directory so they can be found at launch - soDir = os.path.join(rootPath,'app', 'code','linux') - for libPath in glob.iglob(os.path.join(soDir,"*.so")): + # Copy libs to the app directory so they can be found at launch + soDir = os.path.join(rootPath, 'app', 'code', 'linux') + for libPath in glob.iglob(os.path.join(soDir, "*.so")): libFileName = os.path.basename(libPath) - srcPath = os.path.join(soDir,libFileName) - linkPath = os.path.join(dstPath,libFileName) - shutil.copy(srcPath,linkPath) + srcPath = os.path.join(soDir, libFileName) + linkPath = os.path.join(dstPath, libFileName) + shutil.copy(srcPath, linkPath) @@ -90,7 +105,7 @@ def buildDistTree(buildPath, rootPath, args, isStaticRelease): srcPath = os.path.join(appInstallRoot, 'jre', 'linux', jreRelease) dstPath = os.path.join(rootPath, os.path.basename(srcPath)) shutil.copytree(srcPath, dstPath, symlinks=True) - + # Form the executable bash script dstPath = os.path.join(rootPath, 'run' + appName) buildBashScript(dstPath, args, isStaticRelease) @@ -104,6 +119,14 @@ def buildBashScript(destFile, args, isStaticRelease): jvmArgsStr += aStr + ' ' f = open(destFile, 'wb') + f.write('#!/bin/bash\n') + f.write('# Do not remove the opening or closing brackets: {}. This enables safe inline\n') + f.write('# mutations to this script while it is running\n') + f.write('{ # Do not remove this bracket! \n\n') + + f.write('# Define the maximum memory to allow the application to utilize\n') + f.write('#maxMem=512m # Uncomment out this line to change from defaults.\n\n') + f.write('# Get the instalation path\n') f.write('installPath=$(readlink -f "$BASH_SOURCE")\n') f.write('installPath=$(dirname "$installPath")\n\n') @@ -111,14 +134,22 @@ def buildBashScript(destFile, args, isStaticRelease): f.write('# Change the working directory to the app folder in the installation path\n') f.write('cd "$installPath"/app\n\n') - exeCmd = 'java ' + jvmArgsStr - if isStaticRelease == True: - exeCmd = '../' + jreRelease + '/bin/java ' + jvmArgsStr - exeCmd = exeCmd + ' -Djava.system.class.loader=appLauncher.RootClassLoader -cp ../launcher/appLauncher.jar appLauncher.AppLauncher app.cfg' + f.write('# Setup the xmxStr to define the maximum JVM memory.\n') + f.write('if [ -z ${maxMem+x} ]; then\n') + f.write(' xmxStr=""\n') + f.write('else\n') + f.write(' xmxStr=\'-Xmx\'$maxMem\n') + f.write('fi\n\n') + exeCmd = 'java ' + jvmArgsStr + '$xmxStr ' + if isStaticRelease == True: + exeCmd = '../' + args.jreRelease + '/bin/java ' + jvmArgsStr + '$xmxStr ' + exeCmd = exeCmd + '-Djava.system.class.loader=appLauncher.RootClassLoader -cp ../launcher/appLauncher.jar appLauncher.AppLauncher app.cfg ' f.write('# Run the application\n') - f.write(exeCmd + '\n') - f.write('\n') + f.write(exeCmd + '\n\n') + + f.write('exit # Do not remove this exit! (just before the bracket)\n') + f.write('} # Do not remove this bracket! \n\n') f.close() diff --git a/script/logUtils.py b/script/logUtils.py new file mode 100644 index 0000000..042b5e2 --- /dev/null +++ b/script/logUtils.py @@ -0,0 +1,24 @@ +import hashlib +import os +import time +import sys + + +def appendLogOutputWithText(aStr, aText): + """ Method which given a string from a log will insert all occurances of newlines with the + specified text. Note if the last character is a newline then it will be stripped before the replacement""" + if aStr.endswith("\n") == True: + aStr = aStr[:-1] + retStr = aText + aStr.replace('\n', '\n' + aText) + + return retStr; + + +def log(aStr, tabReplace=None): + """ Method which will print aStr to the console and optionally replace all tab + characters with tabReplace""" + if tabReplace != None: + aStr.replace('\t', tabReplace) + + print(aStr) + diff --git a/script/miscUtils.py b/script/miscUtils.py index 013ce0b..e9ab53f 100644 --- a/script/miscUtils.py +++ b/script/miscUtils.py @@ -2,18 +2,77 @@ import hashlib import os import time +import subprocess import sys -def handleSignal(signal, frame): - """Signal handler, typically used to capture ctrl-c.""" - print('User aborted processing!') - sys.exit(0) +import logUtils +def checkRoot(): + """Determines if the script is running with root priveleges.""" + # This logic will may break on SELinux systems + if os.geteuid() != 0: + msg = ' You need to have root privileges to run this script.\n' + msg += ' Please run this script with sudo!\n' + msg += ' Exiting...\n' + exit(msg) + + +# Source: http://stackoverflow.com/questions/1131220/get-md5-hash-of-a-files-without-open-it-in-python +def computeMd5ForFile(evalFile, block_size=2**20): + f = open(evalFile, 'rb') + md5 = hashlib.md5() + while True: + data = f.read(block_size) + if not data: + break + md5.update(data) + f.close() + return md5.hexdigest() + + +def getInstallRoot(): + """Returns the root path where the running script is installed.""" + argv = sys.argv; + installRoot = os.path.dirname(argv[0]) +# print('appInstallRoot: ' + appInstallRoot) + return installRoot + + +def executeAndLog(command, indentStr=""): + """Executes the specified command via subprocess and logs all (stderr,stdout) output to the console""" +# proc = subprocess. +# proc.returncode = -1 + try: + proc = subprocess.Popen(command, stderr=subprocess.STDOUT, stdout=subprocess.PIPE) + proc.wait() + outStr = proc.stdout.read() + if outStr != "": + outStr = logUtils.appendLogOutputWithText(outStr, indentStr) + print(outStr) + except Exception as aExp: + print(indentStr + 'Failed to execute command: ' + str(command)) + outStr = logUtils.appendLogOutputWithText(str(aExp), indentStr) + print(outStr) + print(indentStr + 'Stack Trace:') + outStr = logUtils.appendLogOutputWithText(aExp.child_traceback, indentStr + '\t') + print(outStr) + + class Proc: + returncode = None + proc = Proc + +# proc = None + + return proc +# if proc.returncode != 0: +# print('\tError: Failed to build executable. Return code: ' + proc.returncode) + + def getPathSize(aRoot): """Computes the total disk space used by the specified path. Note if aRoot does not exist or is None then this will return 0""" - # Check for existance + # Check for existence if aRoot == None or os.path.exists(aRoot) == False: return 0 @@ -32,31 +91,10 @@ def getPathSize(aRoot): return numBytes -def checkRoot(): - """ - Determines if the scrip ist running with root priveleges.""" - # This logic will may break on SELinux systems - if os.geteuid() != 0: - msg = ' You need to have root privileges to run this script.\n' - msg += ' Please run this script with sudo!\n' - msg += ' Exiting...\n' - exit(msg) - - -def getInstallRoot(): - """Returns the root path where the running script is insalled.""" - argv = sys.argv; - installRoot = os.path.dirname(argv[0]) -# print('appInstallRoot: ' + appInstallRoot) - return installRoot - - -def isJreAvailable(systemName, jreRelease): - appInstallRoot = getInstallRoot() - appInstallRoot = os.path.dirname(appInstallRoot) - - srcPath = os.path.join(appInstallRoot, 'jre', systemName, jreRelease) - return os.path.isdir(srcPath) +def handleSignal(signal, frame): + """Signal handler, typically used to capture ctrl-c.""" + print('User aborted processing!') + sys.exit(0) def buildAppLauncherConfig(destFile, args): @@ -114,15 +152,3 @@ def buildAppLauncherConfig(destFile, args): f.close() - -# Source: http://stackoverflow.com/questions/1131220/get-md5-hash-of-a-files-without-open-it-in-python -def computeMd5ForFile(evalFile, block_size=2**20): - f = open(evalFile, 'rb') - md5 = hashlib.md5() - while True: - data = f.read(block_size) - if not data: - break - md5.update(data) - f.close() - return md5.hexdigest() \ No newline at end of file diff --git a/script/windowsUtils.py b/script/windowsUtils.py index 30ed226..1dada2d 100644 --- a/script/windowsUtils.py +++ b/script/windowsUtils.py @@ -1,30 +1,38 @@ #! /usr/bin/env python import argparse +import copy import getpass import math import os import shutil import signal -import subprocess import sys import tempfile import time import glob +import jreUtils import miscUtils -# Globals -# The global var jre is a pointer to a full jre release for a Windows system -# There should be a release located at /jre/windows/jreRelease -jreRelease = 'jre1.7.0_15' - - def buildRelease(args, buildPath): + # We mutate args - thus make a custom copy + args = copy.copy(args) + # Retrieve vars of interest appName = args.name version = args.version + jreRelease = args.jreRelease + + # Attempt to locate a default JRE if none is specified in the args. + if jreRelease == None: + jreRelease = jreUtils.getDefaultJreRelease('windows') + # Let the user know if the 'user' specified JRE is not available and locate an alternative + elif jreUtils.isJreAvailable('windows', jreRelease) == False: + print('[Warning] User specified JRE (' + jreRelease + ') is not available for Windows platform. Searching for alternative...') + jreRelease = jreUtils.getDefaultJreRelease('windows') + args.jreRelease = jreRelease # Bail if launch4j is not installed appInstallRoot = miscUtils.getInstallRoot() @@ -39,12 +47,15 @@ def buildRelease(args, buildPath): # Form the list of distributions to build distList = [(appName + '-' + version, False)] - if miscUtils.isJreAvailable('windows', jreRelease) == True: + if jreUtils.isJreAvailable('windows', jreRelease) == True: distList.append((appName + '-' + version + '-jre', True)) # Create the various Windows distributions for (distName, isStaticRelease) in distList: print('Building Windows distribution: ' + distName) + # Let the user know of the JRE release we are going to build with + if isStaticRelease == True: + print('\tUtilizing jreRelease: ' + jreRelease) # Create the (top level) distribution folder dstPath = os.path.join(tmpPath, distName) @@ -55,7 +66,14 @@ def buildRelease(args, buildPath): # Create the zip file zipFile = os.path.join(buildPath, distName + ".zip") - subprocess.check_call(["jar", "-cMf", zipFile, "-C", dstPath, '.'], stderr=subprocess.STDOUT) + cmd = ["jar", "-cMf", zipFile, "-C", dstPath, '.'] + print('\tForming zip file: ' + zipFile) + proc = miscUtils.executeAndLog(cmd, "\t\tjar: ") + if proc.returncode != 0: + print('\tError: Failed to form zip file. Return code: ' + str(proc.returncode)) + print('\tAborting build of release: ' + os.path.basename(zipFile)) + else: + print('\tFinished building release: ' + os.path.basename(zipFile)) # Perform cleanup shutil.rmtree(tmpPath) @@ -66,6 +84,7 @@ def buildDistTree(buildPath, rootPath, args, isStaticRelease): appInstallRoot = miscUtils.getInstallRoot() appInstallRoot = os.path.dirname(appInstallRoot) appName = args.name + jreRelease = args.jreRelease # Form the app contents folder srcPath = os.path.join(buildPath, "delta") @@ -104,11 +123,14 @@ def buildDistTree(buildPath, rootPath, args, isStaticRelease): origIconFile = args.iconFile if origIconFile != None: winIconFile = os.path.join(rootPath, os.path.basename(origIconFile) + ".ico") - eCode = subprocess.call(["convert", origIconFile, winIconFile], stderr=subprocess.STDOUT) - if eCode != 0: - print('ImageMagick convert failed with eCode: ' + str(eCode)) - print('There will be no windows icon file. System call:') - print(' convert ' + origIconFile + ' ' + winIconFile) + cmd = ['convert', origIconFile, winIconFile] + proc = miscUtils.executeAndLog(cmd, "\t\t(ImageMagick) convert: ") + if proc.returncode != 0: + if proc.returncode == None: + print('\t\tImageMagick convert does not appear to be properly installed.') + else: + print('\t\tImageMagick convert failed with eCode: ' + str(proc.returncode)) + print('\t\tThere will be no windows icon file.') winIconFile = None # Form the launch4j config file @@ -117,8 +139,11 @@ def buildDistTree(buildPath, rootPath, args, isStaticRelease): # Build the windows executable launch4jExe = os.path.join(appInstallRoot, "launch4j", "launch4j") - subprocess.check_call([launch4jExe, configFile], stderr=subprocess.STDOUT) -# print(launch4jExe + ' ' + configFile) + cmd = [launch4jExe, configFile] + print('\tBuilding windows executable via launch4j') + proc = miscUtils.executeAndLog(cmd, "\t\t") + if proc.returncode != 0: + print('\tError: Failed to build executable. Return code: ' + str(proc.returncode)) # Perform cleanup os.remove(configFile) @@ -162,9 +187,11 @@ def buildLaunch4JConfig(destFile, args, isStaticRelease, iconFile): writeln(f, 0, ""); writeln(f, 1, ""); if isStaticRelease == True: - writeln(f, 2, "" + jreRelease + ""); + writeln(f, 2, "" + args.jreRelease + ""); else: - writeln(f, 2, "" + "1.7.0" + ""); + jreVer = jreUtils.getJreMajorVersion(args.jreRelease) +# writeln(f, 2, "" + "1.7.0" + ""); + writeln(f, 2, "" + jreVer + ""); # writeln(f, 2, "jreOnly|preferJre|preferJdk|jdkOnly"); writeln(f, 2, "preferJre"); for aJvmArg in args.jvmArgs: diff --git a/src/distMaker/DistMakerEngine.java b/src/distMaker/DistMakerEngine.java index 6529754..3e7a181 100644 --- a/src/distMaker/DistMakerEngine.java +++ b/src/distMaker/DistMakerEngine.java @@ -6,7 +6,7 @@ import glum.gui.panel.task.FullTaskPanel; import glum.io.IoUtil; import glum.net.Credential; import glum.reflect.FunctionRunnable; -import glum.task.Task; +import glum.task.*; import glum.unit.DateUnit; import glum.util.ThreadUtil; @@ -26,9 +26,9 @@ import javax.swing.JFrame; import javax.swing.SwingUtilities; import com.google.common.base.Joiner; -import com.google.common.collect.Lists; import distMaker.gui.PickReleasePanel; +import distMaker.node.FileNode; import distMaker.node.Node; import distMaker.platform.AppleUtils; @@ -118,15 +118,14 @@ public class DistMakerEngine return new UpdateStatus(msg); } // Sort the items, and isolate the newest item - LinkedList fullList = Lists.newLinkedList(unsortedReleaseList); + LinkedList fullList = new LinkedList<>(unsortedReleaseList); Collections.sort(fullList); Release newestRelease = fullList.removeLast(); // The check succeeded, so return wether or not the app is up to date. return new UpdateStatus(newestRelease.equals(currRelease)); } - - + /** * Sets in the credentials used to access the update site. If either argument is null, then the credentials will be * cleared out. @@ -266,9 +265,9 @@ public class DistMakerEngine aTask.abort(); return; } - - // a successful test has been done, so notify the listener - listener.checkForNewVersionsPerformed(); + + // a successful test has been done, so notify the listener + listener.checkForNewVersionsPerformed(); // In case there is only the current version, don't show the update selection panel. // Just show a short message that everything is up to date, and abort. @@ -286,8 +285,6 @@ public class DistMakerEngine } } - - // Hide the taskPanel aTask.setVisible(false); @@ -313,7 +310,7 @@ public class DistMakerEngine chosenItem = pickVersionPanel.getChosenItem(); if (chosenItem == null) return; - + // Reshow the taskPanel aTask.setVisible(true); @@ -379,6 +376,8 @@ public class DistMakerEngine Node staleNode, updateNode; URL catUrl, staleUrl, updateUrl; File catalogFile; + Task mainTask, tmpTask; + long nodeFileLen; boolean isPass; try @@ -393,10 +392,10 @@ public class DistMakerEngine return false; } - // Download the update catalog to the (local) delta location + // Download the update catalog to the (local) delta location (Progress -> [0% - 1%]) catUrl = IoUtil.createURL(updateUrl.toString() + "/catalog.txt"); catalogFile = new File(destPath, "catalog.txt"); - if (DistUtils.downloadFile(aTask, catUrl, catalogFile, refCredential) == false) + if (DistUtils.downloadFile(new PartialTask(aTask, 0.00, 0.01), catUrl, catalogFile, refCredential, -1L) == false) return false; // Load the map of stale nodes @@ -411,43 +410,62 @@ public class DistMakerEngine if (updateMap == null) return false; - // Download the individual files - aTask.infoAppendln("Downloading release: " + aRelease.getVersion() + " Nodes: " + updateMap.size()); + // Determine the total number of bytes to be transferred + long releaseSizeFull = 0L, releaseSizeCurr = 0L; + for (Node aNode : updateMap.values()) + { + if (aNode instanceof FileNode) + releaseSizeFull += ((FileNode)aNode).getFileLen(); + } + + // Download the individual files (Progress -> [1% - 95%]) + double progressVal = 0.00; + mainTask = new PartialTask(aTask, 0.01, 0.94); + mainTask.infoAppendln("Downloading release: " + aRelease.getVersion() + " Nodes: " + updateMap.size()); for (String aFileName : updateMap.keySet()) { // Bail if we have been aborted - if (aTask.isActive() == false) + if (mainTask.isActive() == false) return false; updateNode = updateMap.get(aFileName); staleNode = staleMap.get(aFileName); + nodeFileLen = 0L; + if (updateNode instanceof FileNode) + nodeFileLen = ((FileNode)updateNode).getFileLen(); + tmpTask = new PartialTask(mainTask, mainTask.getProgress(), nodeFileLen / (releaseSizeFull + 0.00)); // Attempt to use the local copy isPass = false; if (staleNode != null && updateNode.areContentsEqual(staleNode) == true) { - isPass = staleNode.transferContentTo(aTask, refCredential, destPath); + isPass = staleNode.transferContentTo(tmpTask, refCredential, destPath); if (isPass == true) - aTask.infoAppendln("\t(L) " + staleNode.getFileName()); + mainTask.infoAppendln("\t(L) " + staleNode.getFileName()); } // Use the remote update copy, if we were not able to use a local stale copy - if (isPass == false && aTask.isActive() == true) + if (isPass == false && mainTask.isActive() == true) { - isPass = updateNode.transferContentTo(aTask, refCredential, destPath); + isPass = updateNode.transferContentTo(tmpTask, refCredential, destPath); if (isPass == true) - aTask.infoAppendln("\t(R) " + updateNode.getFileName()); + mainTask.infoAppendln("\t(R) " + updateNode.getFileName()); } // Log the failure and bail - if (isPass == false && aTask.isActive() == true) + if (isPass == false && mainTask.isActive() == true) { - aTask.infoAppendln("Failed to download from update site."); - aTask.infoAppendln("\tSite: " + updateUrl); - aTask.infoAppendln("\tFile: " + updateNode.getFileName()); - aTask.infoAppendln("\tDest: " + destPath); + mainTask.infoAppendln("Failed to download from update site."); + mainTask.infoAppendln("\tSite: " + updateUrl); + mainTask.infoAppendln("\tFile: " + updateNode.getFileName()); + mainTask.infoAppendln("\tDest: " + destPath); return false; } + + // Update the progress + releaseSizeCurr += nodeFileLen; + progressVal = releaseSizeCurr / (releaseSizeFull + 0.00); + mainTask.setProgress(progressVal); } // Update the platform configuration files diff --git a/src/distMaker/DistUtils.java b/src/distMaker/DistUtils.java index 9eed567..3daac81 100644 --- a/src/distMaker/DistUtils.java +++ b/src/distMaker/DistUtils.java @@ -2,35 +2,19 @@ package distMaker; import glum.gui.GuiUtil; import glum.io.IoUtil; -import glum.net.Credential; -import glum.net.NetUtil; -import glum.net.Result; +import glum.net.*; import glum.reflect.ReflectUtil; import glum.task.ConsoleTask; import glum.task.Task; import glum.unit.DateUnit; import glum.util.ThreadUtil; -import java.io.BufferedInputStream; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; +import java.io.*; import java.net.URL; import java.net.URLConnection; -import java.util.List; -import java.util.Map; +import java.util.*; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - -import distMaker.node.FileNode; -import distMaker.node.Node; -import distMaker.node.PathNode; +import distMaker.node.*; public class DistUtils { @@ -39,10 +23,10 @@ public class DistUtils private static boolean isDevelopersEnvironment = true; private static int updateCode = 0; // Static member to declare the update status, 0: None, 1: Pass, 2: Fail private static String updateMsg = null; - + // Static field used only when developing DistMaker. Otherwise this field should always be null. private static File developAppPath = null; - + /** * Utility method to determine the path where the application is installed. *

@@ -51,7 +35,7 @@ public class DistUtils public static File getAppPath() { File jarPath, currPath, testFile; - + // If we are in developer mode then return the developAppPath if (developAppPath != null) return developAppPath; @@ -101,15 +85,21 @@ public class DistUtils /** * Downloads the specified file from srcUrl to destFile. Returns true on success + *

+ * Note the passed in task's progress will be updated from 0% to 100% at file download completion, If the specified + * file size is invalid (aFileSize <= 0) or the download turns out to be bigger than the specified size then there + * will be no progress update while the file is being downloaded - only at completion. */ @SuppressWarnings("resource") - public static boolean downloadFile(Task aTask, URL aUrl, File aFile, Credential aCredential) + public static boolean downloadFile(Task aTask, URL aUrl, File aFile, Credential aCredential, long aFileSize) { URLConnection connection; InputStream inStream; OutputStream outStream; String errMsg; byte[] byteArr; + double progressVal; + long cntByteFull, cntByteCurr; int numBytes; // Ensure we have a valid aTask @@ -135,21 +125,42 @@ public class DistUtils outStream = new FileOutputStream(aFile); // Copy the bytes from the instream to the outstream + cntByteFull = aFileSize; + cntByteCurr = 0; numBytes = 0; while (numBytes != -1) { numBytes = inStream.read(byteArr); if (numBytes > 0) + { outStream.write(byteArr, 0, numBytes); + cntByteCurr += numBytes; + } + + // Update the progressVal to reflect the download progress. Note however that we do update the + // progress to 100% since that would change the task to be flagged as inactive and thus cause + // the download to be aborted prematurely. + // TODO: In the future Tasks should not be marked as inactive based on progress values + progressVal = 0; + if (cntByteFull > 0) + { + progressVal = (cntByteCurr + 0.0) / cntByteFull; + if (progressVal >= 1.0) + progressVal = 0.99; + } + aTask.setProgress(progressVal); // Bail if aTask is aborted if (aTask.isActive() == false) { aTask.infoAppendln("File transfer request has been aborted..."); - aTask.infoAppendln("\tFile: " + aFile); + aTask.infoAppendln("\tFile: " + aFile + " Bytes transferred: " + cntByteCurr); return false; } } + + // Mark aTask's progress as complete since the file was downloaded. + aTask.setProgress(1.0); } catch (IOException aExp) { @@ -180,7 +191,7 @@ public class DistUtils String errMsg; errMsg = null; - fullList = Lists.newArrayList(); + fullList = new ArrayList<>(); catUrl = IoUtil.createURL(aUpdateUrl.toString() + "/" + appName + "/" + "releaseInfo.txt"); connection = null; @@ -259,7 +270,7 @@ public class DistUtils URL fetchUrl; Result result; String errMsg; - + // Dump the stack trace aExp.printStackTrace(); @@ -291,8 +302,8 @@ public class DistUtils errMsg += "An undefined error occurred while retrieving the remote file.\n"; break; } - - // Log the URL which we failed on + + // Log the URL which we failed on fetchUrl = aConnection.getURL(); errMsg += "\tURL: " + fetchUrl + "\n"; @@ -307,12 +318,12 @@ public class DistUtils public static boolean isFullyWriteable(File aPath) { // There is no known way to change, the write bit to true, in windows, - // so by default, assume the path is writable. This method is totally unreliable on - // the Windows platform (Gives bogus results for files on CDs). + // so by default, assume the path is writable. This method is totally unreliable on + // the Windows platform (Gives bogus results for files on CDs). // TODO: See if File.canWrite(), returns the proper value on Windows if (System.getProperty("os.name").startsWith("Windows") == true) return true; - + if (aPath.isDirectory() == false) throw new RuntimeException("Specified path is not a folder: " + aPath); @@ -342,7 +353,7 @@ public class DistUtils String errMsg; errMsg = null; - retMap = Maps.newLinkedHashMap(); + retMap = new LinkedHashMap<>(); inStream = null; bufReader = null; diff --git a/src/distMaker/LoggingTask.java b/src/distMaker/LoggingTask.java index 165aea9..ce9ac0a 100644 --- a/src/distMaker/LoggingTask.java +++ b/src/distMaker/LoggingTask.java @@ -1,14 +1,13 @@ package distMaker; -import java.util.List; - -import com.google.common.collect.Lists; - import glum.task.SilentTask; +import java.util.ArrayList; +import java.util.List; + public class LoggingTask extends SilentTask { - private final List messages = Lists.newArrayList(); + private final List messages = new ArrayList<>(); @Override public void infoAppend(String aMsg) diff --git a/src/distMaker/gui/MemoryConfigPanel.java b/src/distMaker/gui/MemoryConfigPanel.java index 2c1a92d..f4b3934 100644 --- a/src/distMaker/gui/MemoryConfigPanel.java +++ b/src/distMaker/gui/MemoryConfigPanel.java @@ -30,6 +30,9 @@ import static distMaker.platform.MemUtils.GB_SIZE; public class MemoryConfigPanel extends GlassPanel implements ActionListener, ZioRaw, ListSelectionListener { + /** Unused - but added to eliminate warning due to poorly designed java.io.Serializable interface. */ + private static final long serialVersionUID = 1L; + // GUI vars private JLabel titleL; private GLabel maxMemL, currMemL, targMemL; diff --git a/src/distMaker/gui/PickReleasePanel.java b/src/distMaker/gui/PickReleasePanel.java index 58f4ea4..193e158 100644 --- a/src/distMaker/gui/PickReleasePanel.java +++ b/src/distMaker/gui/PickReleasePanel.java @@ -6,37 +6,24 @@ import glum.gui.action.ClickAction; import glum.gui.panel.GlassPanel; import glum.gui.panel.itemList.ItemListPanel; import glum.gui.panel.itemList.StaticItemProcessor; -import glum.gui.panel.itemList.query.QueryComposer; -import glum.gui.panel.itemList.query.QueryItemHandler; -import glum.gui.panel.itemList.query.QueryTableCellRenderer; +import glum.gui.panel.itemList.query.*; import glum.unit.ConstUnitProvider; import glum.unit.DateUnit; import glum.zio.raw.ZioRaw; -import java.awt.Component; -import java.awt.Dimension; -import java.awt.Font; +import java.awt.*; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; -import java.util.Collections; -import java.util.LinkedList; +import java.util.*; import java.util.List; -import javax.swing.JButton; -import javax.swing.JLabel; -import javax.swing.JPanel; -import javax.swing.JRadioButton; -import javax.swing.JTextArea; -import javax.swing.JTextField; +import javax.swing.*; import javax.swing.border.BevelBorder; import javax.swing.border.EmptyBorder; import javax.swing.event.ListSelectionEvent; import javax.swing.event.ListSelectionListener; import net.miginfocom.swing.MigLayout; - -import com.google.common.collect.Lists; - import distMaker.LookUp; import distMaker.Release; @@ -97,8 +84,8 @@ public class PickReleasePanel extends GlassPanel implements ActionListener, ZioR String appName, infoMsg; // Sort the items, and isolate the newest item - LinkedList linkedList; - linkedList = Lists.newLinkedList(itemList); + LinkedList linkedList; + linkedList = new LinkedList<>(itemList); Collections.sort(linkedList); Collections.reverse(linkedList); // reverse the list to show most recent versions on top newestItem = linkedList.removeFirst(); diff --git a/src/distMaker/node/FileNode.java b/src/distMaker/node/FileNode.java index e3881f1..4d61880 100644 --- a/src/distMaker/node/FileNode.java +++ b/src/distMaker/node/FileNode.java @@ -43,6 +43,14 @@ public class FileNode implements Node return true; } + /** + * Returns the length of the associated file + */ + public long getFileLen() + { + return fileLen; + } + @Override public String getFileName() { @@ -57,10 +65,10 @@ public class FileNode implements Node // Determine the source URL to copy the contents from srcUrl = IoUtil.createURL(rootUrl.toString() + "/" + fileName); - + // Determine the file to transfer the contents to dstFile = new File(dstPath, fileName); - return DistUtils.downloadFile(aTask, srcUrl, dstFile, aCredential); + return DistUtils.downloadFile(aTask, srcUrl, dstFile, aCredential, fileLen); } } diff --git a/src/distMaker/platform/LinuxUtils.java b/src/distMaker/platform/LinuxUtils.java index a02cb3a..661d204 100644 --- a/src/distMaker/platform/LinuxUtils.java +++ b/src/distMaker/platform/LinuxUtils.java @@ -3,29 +3,32 @@ package distMaker.platform; import glum.io.IoUtil; import java.io.*; +import java.nio.file.Files; +import java.util.ArrayList; import java.util.List; -import com.google.common.collect.Lists; +import distMaker.DistUtils; public class LinuxUtils { - /** - * Utility method to update the specified max memory (-Xmx) value in the text file (aFile) to the specified - * maxMemVal. + * Utility method to update the specified maxMem var in the script (aFile) to the requested number of bytes. *

- * Note this method is very brittle, and assumes that there is a single value where the string, -Xmx, is specified in - * the script. It assumes this string will be surrounded by a single space character on each side. + * Note this method assumes the specified file is a shell script built by DistMaker where the var maxMem holds the proper (right side) specification for the + * JVM's -Xmx value. + *

+ * If the maxMem var definition is moved in the script file to after the launch of the application then this method will (silently) fail to configure the + * value needed to launch the JVM. */ public static boolean updateMaxMem(File aFile, long numBytes) { BufferedReader br; List inputList; - String strLine, updateStr; - boolean isProcessed; + String evalStr, memStr, tmpStr; + int currLineNum, injectLineNum, targLineNum; - inputList = Lists.newArrayList(); - isProcessed = false; + inputList = new ArrayList<>(); + int zios_clean; // Process our input br = null; @@ -34,23 +37,27 @@ public class LinuxUtils br = new BufferedReader(new InputStreamReader(new FileInputStream(aFile))); // Read the lines + currLineNum = 0; + targLineNum = -1; + injectLineNum = -1; while (true) { - strLine = br.readLine(); - if (strLine == null) + evalStr = br.readLine(); + if (evalStr == null) break; - updateStr = MemUtils.transformMaxMemHeapString(strLine, numBytes); - if (updateStr != null) - { - isProcessed = true; - strLine = updateStr; - } + // Locate where we should place our maxMem configuration var + tmpStr = evalStr.trim(); + if (tmpStr.equals("# Define the maximum memory to allow the application to utilize") == true) + injectLineNum = currLineNum + 1; + else if (tmpStr.startsWith("maxMem=") == true) + targLineNum = currLineNum; - inputList.add(strLine); + inputList.add(evalStr); + currLineNum++; } } - catch (Exception aExp) + catch(Exception aExp) { aExp.printStackTrace(); return false; @@ -60,13 +67,21 @@ public class LinuxUtils IoUtil.forceClose(br); } - // Bail if we did not find a line to change - if (isProcessed == false) + // Determine the memStr to use + if (numBytes % MemUtils.GB_SIZE == 0) + memStr = "" + (numBytes / MemUtils.GB_SIZE) + "g"; + else + memStr = "" + (numBytes / MemUtils.MB_SIZE) + "m"; + + // Insert our changes into the script + if (targLineNum != -1) + inputList.set(targLineNum, "maxMem=" + memStr); + else if (injectLineNum != -1 && injectLineNum < currLineNum) + inputList.add(injectLineNum + 1, "maxMem=" + memStr); + else { - Exception aExp; - aExp = new Exception("Failed to locate -Xmx string!"); - aExp.printStackTrace(); - return false; + inputList.add(0, "# Define the maximum memory to allow the application to utilize"); + inputList.add(1, "maxMem=" + memStr + "\n"); } // Update the script @@ -74,6 +89,41 @@ public class LinuxUtils return writeDoc(aFile, inputList); } + /** + * Returns the executable script used to launch the JVM. If one can not be determined then this method will return null. + *

+ * If there are multiple launch scripts then this method may grab the wrong file and fail. + *

+ * TODO: In the future the launch script should pass itself as an argument to the JVM and DistMaker should keep track of that. If the script is significantly + * manipulated from the original the launch file may be improperly detected. + */ + public static File getScriptFile() + { + File[] fileArr; + File installPath; + File retFile; + + installPath = DistUtils.getAppPath().getParentFile(); + fileArr = installPath.listFiles(); + + // Attempt to locate the path that matches run* file + retFile = null; + for (File aFile : fileArr) + { + if (aFile.getName().startsWith("run") == true) + retFile = aFile; + } + + if (retFile == null) + return null; + + if (retFile.isFile() == false && Files.isExecutable(retFile.toPath()) == false) + return null; + + System.out.println("Linux launch file: " + retFile); + return retFile; + } + /** * Helper method to output the specified strings to aFile */ @@ -91,7 +141,7 @@ public class LinuxUtils for (String aStr : strList) bw.write(aStr + '\n'); } - catch (Exception aExp) + catch(Exception aExp) { aExp.printStackTrace(); return false; diff --git a/src/distMaker/platform/MemUtils.java b/src/distMaker/platform/MemUtils.java index 35a2944..f129a40 100644 --- a/src/distMaker/platform/MemUtils.java +++ b/src/distMaker/platform/MemUtils.java @@ -1,10 +1,13 @@ package distMaker.platform; import glum.gui.panel.generic.MessagePanel; +import glum.reflect.ReflectUtil; import glum.unit.ByteUnit; import java.io.File; import java.lang.management.ManagementFactory; +import java.lang.management.OperatingSystemMXBean; +import java.lang.reflect.Method; import distMaker.DistUtils; @@ -19,22 +22,33 @@ public class MemUtils * Utility method that attempts to compute the installed system memory (ram). If the installed system ram can not be * computed, then the system is assumed to have 4 GB. */ - @SuppressWarnings("restriction") public static long getInstalledSystemMemory() { ByteUnit byteUnit; long systemMem; - // Attempt to interogate the system memory using the Sun/Oracle JVM specific method + // Attempt to interrogate the system memory using the Sun/Oracle JVM specific method try { - systemMem = ((com.sun.management.OperatingSystemMXBean)ManagementFactory.getOperatingSystemMXBean()).getTotalPhysicalMemorySize(); + OperatingSystemMXBean osBean; + Class tmpClass; + Method tmpMethod; + + // Note we gather the available memory via reflection to avoid Eclipse/IDE compilation errors due + // to restricted com.sun.management package. The commented out single line of code is equivalent to + // the next block (4 lines) of code. + // Retrieve the OS available memory via reflection equivalent to the commented method below: +// systemMem = ((com.sun.management.OperatingSystemMXBean)ManagementFactory.getOperatingSystemMXBean()).getTotalPhysicalMemorySize(); + osBean = ManagementFactory.getOperatingSystemMXBean(); + tmpClass = Class.forName("com.sun.management.OperatingSystemMXBean"); + tmpMethod = ReflectUtil.locateMatchingMethod(tmpClass, "getTotalPhysicalMemorySize"); + systemMem = (Long)tmpMethod.invoke(osBean); byteUnit = new ByteUnit(2); System.out.println("Max memory on the system: " + byteUnit.getString(systemMem)); return systemMem; } - catch (Throwable aThrowable) + catch(Throwable aThrowable) { System.out.println("Failed to query the installed system memory! Assume system memory is 4 GB."); System.out.println("Exception: " + aThrowable.getLocalizedMessage()); @@ -91,8 +105,8 @@ public class MemUtils } // Linux specific platform files - scriptFile = new File(installPath, "runEcho"); - if (scriptFile.isFile() == true) + scriptFile = LinuxUtils.getScriptFile(); + if (scriptFile != null && scriptFile.isFile() == true) { isValidPlatform = true; diff --git a/src/distMaker/platform/WindowsUtils.java b/src/distMaker/platform/WindowsUtils.java index 0652712..491875e 100644 --- a/src/distMaker/platform/WindowsUtils.java +++ b/src/distMaker/platform/WindowsUtils.java @@ -3,10 +3,9 @@ package distMaker.platform; import glum.io.IoUtil; import java.io.*; +import java.util.ArrayList; import java.util.List; -import com.google.common.collect.Lists; - import distMaker.DistUtils; public class WindowsUtils @@ -25,7 +24,7 @@ public class WindowsUtils String strLine, updateStr; boolean isProcessed; - inputList = Lists.newArrayList(); + inputList = new ArrayList<>(); isProcessed = false; // Process our input @@ -50,7 +49,7 @@ public class WindowsUtils inputList.add(strLine); } - + // Create a new max heap input config line if one is not specified if (isProcessed == false) { @@ -60,7 +59,7 @@ public class WindowsUtils isProcessed = true; } } - catch (Exception aExp) + catch(Exception aExp) { aExp.printStackTrace(); return false; @@ -80,8 +79,9 @@ public class WindowsUtils *

* If the configuration file is determined but does not exist, then an empty configuration file will be created. *

- * Note this method looks for a file that ends in .l4j.cfg, or an exe file and creates the corresponding config file. - *

If there are multiple .exe or .l4j.cfg files, then this method may grab the wrong file and fail. + * Note this method looks for a file that ends in .l4j.cfg, or an exe file and creates the corresponding config file. + *

+ * If there are multiple .exe or .l4j.cfg files, then this method may grab the wrong file and fail. */ public static File getConfigFile() { @@ -89,10 +89,9 @@ public class WindowsUtils File installPath; File retFile; - installPath = DistUtils.getAppPath().getParentFile(); fileArr = installPath.listFiles(); - + // Attempt to locate the .l4j.ini file retFile = null; for (File aFile : fileArr) @@ -100,7 +99,7 @@ public class WindowsUtils if (aFile.getName().endsWith(".l4j.ini") == true) retFile = aFile; } - + if (retFile == null) { for (File aFile : fileArr) @@ -109,10 +108,10 @@ public class WindowsUtils retFile = new File(aFile.getParentFile(), aFile.getName().substring(0, aFile.getName().length() - 4) + ".l4j.ini"); } } - + if (retFile == null) return null; - + if (retFile.isFile() == false) { try diff --git a/src/dsstore/AliasRecord.java b/src/dsstore/AliasRecord.java index 97c1257..d84b1bc 100644 --- a/src/dsstore/AliasRecord.java +++ b/src/dsstore/AliasRecord.java @@ -1,10 +1,9 @@ package dsstore; import java.nio.ByteBuffer; +import java.util.LinkedHashMap; import java.util.Map; -import com.google.common.collect.Maps; - import dsstore.ext.*; public class AliasRecord @@ -67,7 +66,7 @@ public class AliasRecord public AliasRecord() { - extMap = Maps.newLinkedHashMap(); + extMap = new LinkedHashMap<>(); } /** @@ -204,13 +203,11 @@ public class AliasRecord tmpStr = "/Volumes/" + aVolName; extMap.put(19, new StrExtInfo(19, tmpStr)); - // TODO: // Remove the (unknown) ExtInfo type == 20 // extMap.remove(20); //System.out.println("Will not remove unknown type: 20\n\t" + extMap.get(20)); - // Update the internal copy of the record size recSize = size(); } diff --git a/src/dsstore/BlockDir.java b/src/dsstore/BlockDir.java index f355d0c..61b8766 100644 --- a/src/dsstore/BlockDir.java +++ b/src/dsstore/BlockDir.java @@ -9,7 +9,6 @@ public class BlockDir private String name; private int index; - public BlockDir() { name = null; @@ -26,26 +25,24 @@ public class BlockDir { return index; } - + public void readData(ByteBuffer srcBuf) { int numBytes; byte[] byteArr; - + numBytes = srcBuf.get() & 0xFF; - + // Directories name byteArr = new byte[numBytes]; srcBuf.get(byteArr); name = new String(byteArr, Charsets.US_ASCII); - + // Block's address index = srcBuf.getInt(); - + // Debug output // System.out.println("[BlockDir] Name: " + name + " index: " + index); } - - - + } diff --git a/src/dsstore/MainApp.java b/src/dsstore/MainApp.java index 1d98ffb..429e932 100644 --- a/src/dsstore/MainApp.java +++ b/src/dsstore/MainApp.java @@ -6,11 +6,10 @@ import glum.zio.*; import java.io.*; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Collections; import java.util.List; -import com.google.common.collect.Lists; - import dsstore.record.*; /** @@ -21,7 +20,7 @@ import dsstore.record.*; public class MainApp { // Constants - public static final String DT_long = "long";//convertStringToInt("long"); // An integer (4 bytes) + public static final String DT_long = "long"; //convertStringToInt("long"); // An integer (4 bytes) public static final String DT_shor = "shor"; //convertStringToInt("shor"); // A short integer? Still stored as four bytes, but the first two are always zero. public static final String DT_bool = "bool"; //convertStringToInt("bool"); // A boolean value, stored as one byte. public static final String DT_blob = "blob"; //convertStringToInt("blob"); // An arbitrary block of bytes, stored as an integer followed by that many bytes of data. @@ -32,7 +31,7 @@ public class MainApp private Task refTask; private ByteBuffer dataBuf; private String volumeName; - + public MainApp(String aVolumeName) { // Only output verbose info if we are not updating a store @@ -43,52 +42,55 @@ public class MainApp dataBuf = null; volumeName = aVolumeName; } - + public void writeStore(File aFile) { FileZoutStream aStream; int fileMagicKey; - + // Ensure we have a valid dataBuf if (dataBuf == null) return; - + aStream = null; try { aStream = new FileZoutStream(aFile); - + // Write the file's MagicKey fileMagicKey = 0x0001; aStream.writeInt(fileMagicKey); // Dump the contents of aBytBuff aStream.writeFully(dataBuf.array()); - + aStream.close(); } - catch (IOException aExp) + catch(IOException aExp) { IoUtil.forceClose(aStream); aExp.printStackTrace(); } - - } - - - - public ByteBuffer readStore(File aFile) + } + + /** + * Method to read the actual store contents. + * + * @return True if we successfully read the store + */ + public boolean readStore(File aFile) { ZinStream aStream; int fileSize; + // Bail if the file is not valid if (aFile.isFile() == false) { - refTask.infoAppendln("File does note exist: " + aFile); - return null; + System.err.println("File does note exist: " + aFile); + return false; } dataBuf = null; @@ -168,7 +170,7 @@ refTask.infoAppendln("BlockAddr[" + c1 + "] -> Size: " + blkLen + " Offset: " + dirCnt = dataBuf.getInt(); refTask.infoAppendln("Block directory count: " + dirCnt); - blockDirList = Lists.newLinkedList(); + blockDirList = new ArrayList<>(dirCnt); for (int c1 = 0; c1 < dirCnt; c1++) blockDirList.add(new BlockDir(dataBuf)); @@ -208,9 +210,10 @@ refTask.infoAppendln("Reading freelists..."); { aExp.printStackTrace(); IoUtil.forceClose(aStream); + return false; } - return dataBuf; + return true; } @@ -230,7 +233,7 @@ refTask.infoAppendln("Reading freelists..."); numRecords = aBuf.getInt(); bkgdRecord = null; - recordList = Lists.newLinkedList(); + recordList = new ArrayList<>(numRecords); for (int c1 = 0; c1 < numRecords; c1++) { BufUtils.seek(aBuf, 2); // empty @@ -317,7 +320,7 @@ refTask.infoAppendln("Reading freelists..."); int numRecords; int numNodes; int valConst; - + // Read the DSDB header // 0: The block number of the root node of the B-tree // 1: The number of levels of internal nodes (tree height minus one --- that is, for a tree containing only a single, leaf, node this will be zero) @@ -329,7 +332,7 @@ refTask.infoAppendln("Reading freelists..."); numRecords = srcBuf.getInt(); numNodes = srcBuf.getInt(); valConst = srcBuf.getInt(); - + refTask.infoAppendln("rootBlockNum: " + rootBlockNum); refTask.infoAppendln("numLevels: " + numLevels); refTask.infoAppendln("rootRecords: " + numRecords); @@ -338,7 +341,6 @@ refTask.infoAppendln("Reading freelists..."); } - /** * Application main entry point */ @@ -347,13 +349,14 @@ refTask.infoAppendln("Reading freelists..."); MainApp aMainApp; File aFile; String storeFileName, newVolName; - + storeFileName = null; newVolName = null; - + if (args.length == 0) { System.out.println("Usage: dsStoreUtil "); + System.exit(-1); return; } if (args.length >= 1) @@ -364,13 +367,16 @@ refTask.infoAppendln("Reading freelists..."); { newVolName = args[1]; } - + aFile = new File(storeFileName); - System.out.println("\tUpdating store: " + aFile); - + System.out.println("Updating store: " + aFile); + aMainApp = new MainApp(newVolName); - aMainApp.readStore(aFile); - + + // Bail if we failed to read the store + if (aMainApp.readStore(aFile) == false) + System.exit(-1); + // Save the ds_store if there was a request for a new name if (newVolName != null) aMainApp.writeStore(aFile);