AtHeartEngineer/kaiju
1
0
Fork 0
mirror of https://github.com/JHUAPL/kaiju.git synced 2026-01-08 22:58:05 -05:00
Code Issues Packages Projects Releases Wiki Activity

More changes from Jeff.

Browse Source
This commit is contained in:
Eric Winter
2025-08-25 08:17:25 -06:00
parent 3e4c2940e5
commit 8a553afc92
8 changed files with 56 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
testingScripts/buildTest.py
View File

@@ -378,13 +378,12 @@ def main():
)
if debug:
print(f"slack_response_summary = {slack_response_summary}")
# Also write a summary file to the root folder of this test
with open(os.path.join(MAGE_TEST_SET_ROOT, 'testSummary.out'), 'w',
encoding='utf-8') as f:
with open(os.path.join(MAGE_TEST_SET_ROOT,'testSummary.out'), 'w', encoding='utf-8') as f:
f.write(test_report_details_string)
f.write('\n')
# ------------------------------------------------------------------------
if debug:

16
testingScripts/compare_mage_runs_numerical.py
View File

@@ -41,6 +41,9 @@ from kaipy import kaiH5
# Program description.
DESCRIPTION = "Compare MAGE model runs numerically."
# Root of directory tree for this set of tests.
MAGE_TEST_SET_ROOT = os.environ['MAGE_TEST_SET_ROOT']
# Strings to represent test pass and fail.
TEST_PASS = "PASS"
TEST_FAIL = "FAIL"
@@ -114,7 +117,7 @@ def compare_GAMERA_results(runxml1: str, runxml2: str, verbose: bool = False):
f"h5diff --exclude-attribute {step_path} {file1} {file2} "
f"{step_path}"
)
cproc = subprocess.run(cmd, shell=True)
cproc = subprocess.run(cmd, shell=True, check=True)
if cproc.returncode != 0:
return TEST_FAIL
@@ -175,7 +178,7 @@ def compare_REMIX_results(runxml1: str, runxml2: str, verbose: bool = False):
f"h5diff --exclude-attribute {step_path} {file1} {file2} "
f"{step_path}"
)
cproc = subprocess.run(cmd, shell=True)
cproc = subprocess.run(cmd, shell=True, check=True)
if cproc.returncode != 0:
return TEST_FAIL
@@ -236,7 +239,7 @@ def compare_VOLTRON_results(runxml1: str, runxml2: str, verbose: bool = False):
f"h5diff --exclude-attribute {step_path} {file1} {file2} "
f"{step_path}"
)
cproc = subprocess.run(cmd, shell=True)
cproc = subprocess.run(cmd, shell=True, check=True)
if cproc.returncode != 0:
return TEST_FAIL
@@ -342,7 +345,12 @@ def compare_mage_runs_numerical(args: dict):
slack_client, test_report_details_string, thread_ts=thread_ts,
is_test=test
)
# Also write a summary file to the root folder of this test
with open(os.path.join(MAGE_TEST_SET_ROOT,'testSummary.out'), 'w', encoding='utf-8') as f:
f.write(test_report_details_string)
f.write('\n')
# ------------------------------------------------------------------------
if debug:

4
testingScripts/derecho_configuration_check.py
View File

@@ -176,8 +176,8 @@ def derecho_configuration_check(args: dict):
pbs_options["job_priority"] = os.environ["DERECHO_TESTING_PRIORITY"]
pbs_options["walltime"] = "08:00:00"
pbs_options["modules"] = module_names
# pbs_options["condarc"] = os.environ["CONDARC"]
# pbs_options["conda_envs_path"] = os.environ["CONDA_ENVS_PATH"]
pbs_options["condarc"] = os.environ["CONDARC"]
pbs_options["conda_envs_path"] = os.environ["CONDA_ENVS_PATH"]
pbs_options["conda_environment"] = os.environ["CONDA_ENVIRONMENT"]
pbs_options["mage_test_root"] = os.environ["MAGE_TEST_ROOT"]
pbs_options["mage_test_set_root"] = os.environ["MAGE_TEST_SET_ROOT"]

2
testingScripts/intelCheckSubmitReport-template.pbs
View File

@@ -4,7 +4,7 @@
#PBS -A {{ account }}
#PBS -q {{ queue }}
#PBS -l job_priority={{ job_priority }}
#PBS -l walltime=00:20:00
#PBS -l walltime=02:00:00
#PBS -l select=1:ncpus=128:ompthreads=128
#PBS -j oe
#PBS -m abe

8
testingScripts/intelChecks.py
View File

@@ -4,9 +4,9 @@
This script runs a series of tests of the MAGE software using Intel tools.
The Intel Inspector memory checks run in about 2 hours on two derecho nodes.
The Intel Inspector memory checks run in about an hour on two derecho nodes.
The Intel Inspector thread checks run in about 2.5 hours on two derecho nodes.
The Intel Inspector thread checks run in about 45 minutes on two derecho nodes.
The report script runs in about 2 minutes on one derecho node.
@@ -301,7 +301,7 @@ def intelChecks(args: dict):
from_path = os.path.join(TEST_SCRIPTS_DIRECTORY, filename)
to_path = os.path.join(".", filename)
shutil.copyfile(from_path, to_path)
# Generate bcwind data file.
if verbose:
print("Creating bcwind data file.")
@@ -321,7 +321,7 @@ def intelChecks(args: dict):
continue
if debug:
print(f"cproc = {cproc}")
# Generate the LFM grid file.
if verbose:
print("Creating LFM grid file.")

7
testingScripts/intelChecksReport.py
View File

@@ -258,13 +258,12 @@ def main():
)
if debug:
print(f"slack_response_summary = {slack_response_summary}")
# Also write a summary file to the root folder of this test
with open(os.path.join(MAGE_TEST_SET_ROOT, 'testSummary.out'), 'w',
encoding='utf-8') as f:
with open(os.path.join(MAGE_TEST_SET_ROOT,'testSummary.out'), 'w', encoding='utf-8') as f:
f.write(test_report_details_string)
f.write('\n')
# ------------------------------------------------------------------------
if debug:

78
testingScripts/mage_reproducibility_check.py
View File

@@ -9,6 +9,7 @@ Authors
-------
Jeff Garretson
Eric Winter
"""
@@ -39,17 +40,6 @@ DEFAULT_MODULE_SET_FILE = os.path.join(
KAIJUHOME, "testingScripts", "mage_build_test_modules", "intel_mpich.lst"
)
# Default values for command-line arguments when none are supplied (such as
# when this code is called by external code).
args_default = {
"debug": False,
"loud": False,
"slack_on_fail": False,
"test": False,
"verbose": False,
"module_set_file": DEFAULT_MODULE_SET_FILE,
}
# Root of directory tree for this set of tests.
MAGE_TEST_SET_ROOT = os.environ["MAGE_TEST_SET_ROOT"]
@@ -63,6 +53,11 @@ REPRODUCIBILITY_CHECK_DIRECTORY_PREFIX = "reproducibility_check_"
# Path to directory containing the test scripts
TEST_SCRIPTS_DIRECTORY = os.path.join(KAIJUHOME, "testingScripts")
# List of weekly dash test files to copy
WEEKLY_DASH_TEST_FILES = [
"weeklyDashGo.xml",
]
# Path to jinja2 template file for PBS script for build job.
BUILD_MAGE_PBS_TEMPLATE_FILE = os.path.join(
TEST_SCRIPTS_DIRECTORY, "build_mage-template.pbs"
@@ -80,11 +75,6 @@ RUN_MAGE_PBS_TEMPLATE_FILE = os.path.join(
RUN1_MAGE_PBS_SCRIPT = "run1_mage.pbs"
RUN2_MAGE_PBS_SCRIPT = "run2_mage.pbs"
# List of weekly dash test files to copy
WEEKLY_DASH_TEST_FILES = [
"weeklyDashGo.xml",
]
# Path to jinja2 template file for PBS script for comparison.
MAGE_REPRODUCIBILITY_CHECK_PBS_TEMPLATE_FILE = os.path.join(
TEST_SCRIPTS_DIRECTORY, "mage_reproducibility_check-template.pbs"
@@ -236,7 +226,8 @@ def mage_reproducibility_check(args: dict):
# Submit the job.
cmd = f"qsub {BUILD_MAGE_PBS_SCRIPT}"
cproc = subprocess.run(cmd, shell=True, text=True, capture_output=True)
cproc = subprocess.run(cmd, shell=True, check=True, text=True,
capture_output=True)
jobid_build = cproc.stdout.split(".")[0]
# ------------------------------------------------------------------------
@@ -255,6 +246,7 @@ def mage_reproducibility_check(args: dict):
cda2wind_cmd = (
"cda2wind -t0 2016-08-09T02:00:00 -t1 2016-08-09T12:00:00"
)
genRaiju_cmd = "genRAIJU"
mpiexec_cmd = f"mpiexec {KAIJUHOME}/scripts/preproc/pinCpuCores.sh"
voltron_cmd = "../bin/voltron_mpi.x weeklyDashGo.xml"
@@ -277,15 +269,18 @@ def mage_reproducibility_check(args: dict):
pbs_options["job_priority"] = os.environ["DERECHO_TESTING_PRIORITY"]
pbs_options["walltime"] = "08:00:00"
pbs_options["modules"] = module_names
pbs_options["mage_test_root"] = os.environ["MAGE_TEST_ROOT"]
pbs_options["mage_test_set_root"] = os.environ["MAGE_TEST_SET_ROOT"]
pbs_options["conda_environment"] = os.environ["CONDA_ENVIRONMENT"]
pbs_options["kaijuhome"] = KAIJUHOME
pbs_options["tmpdir"] = os.environ["TMPDIR"]
pbs_options["slack_bot_token"] = os.environ["SLACK_BOT_TOKEN"]
pbs_options["mage_test_root"] = os.environ["MAGE_TEST_ROOT"]
pbs_options["mage_test_set_root"] = os.environ["MAGE_TEST_SET_ROOT"]
pbs_options["branch_or_commit"] = os.environ["BRANCH_OR_COMMIT"]
pbs_options["cmake_cmd"] = cmake_cmd
pbs_options["make_cmd"] = make_cmd
pbs_options["genLFM_cmd"] = genLFM_cmd
pbs_options["cda2wind_cmd"] = cda2wind_cmd
pbs_options["genRaiju_cmd"] = genRaiju_cmd
pbs_options["mpiexec_cmd"] = mpiexec_cmd
pbs_options["voltron_cmd"] = voltron_cmd
@@ -355,12 +350,12 @@ def mage_reproducibility_check(args: dict):
pbs_options["job_priority"] = os.environ["DERECHO_TESTING_PRIORITY"]
pbs_options["walltime"] = "02:00:00"
pbs_options["modules"] = module_names
pbs_options["mage_test_root"] = os.environ["MAGE_TEST_ROOT"]
pbs_options["mage_test_set_root"] = os.environ["MAGE_TEST_SET_ROOT"]
pbs_options["conda_environment"] = os.environ["CONDA_ENVIRONMENT"]
pbs_options["kaijuhome"] = KAIJUHOME
pbs_options["tmpdir"] = os.environ["TMPDIR"]
pbs_options["slack_bot_token"] = os.environ["SLACK_BOT_TOKEN"]
pbs_options["mage_test_root"] = os.environ["MAGE_TEST_ROOT"]
pbs_options["mage_test_set_root"] = os.environ["MAGE_TEST_SET_ROOT"]
pbs_options["branch_or_commit"] = os.environ["BRANCH_OR_COMMIT"]
pbs_options["xml1"] = os.path.join(build_directory, "run1",
"weeklyDashGo.xml")
@@ -425,44 +420,31 @@ def mage_reproducibility_check(args: dict):
if debug:
print(f"Ending {sys.argv[0]} at {datetime.datetime.now()}")
# Return nominal status.
return 0
def main():
"""Main program code for the command-line version of this file.
This is the main program code for the command-line version of this file.
It processes command-line options, then calls the primary code.
Parameters
----------
None
Returns
-------
None
Raises
------
None
"""
"""Driver for command-line version of code."""
# Set up the command-line parser.
parser = create_command_line_parser()
# # Add additional arguments specific to this script.
# parser.add_argument(
# "--module_set_file", "-f", default=DEFAULT_MODULE_SET_FILE,
# help=(
# "Path to text file containing set of modules to build with "
# "(default: %(default)s)"
# )
# )
# Parse the command-line arguments.
args = parser.parse_args()
if args.debug:
print(f"args = {args}")
# ------------------------------------------------------------------------
# Convert the arguments from Namespace to dict.
args = vars(args)
# Call the main program logic. Note that the Namespace object (args)
# returned from the option parser is converted to a dict using vars().
mage_reproducibility_check(vars(args))
# Exit normally.
sys.exit(0)
# Pass the command-line arguments to the main function as a dict.
mage_reproducibility_check(args)
if __name__ == "__main__":

2
testingScripts/unitTest.py
View File

@@ -444,7 +444,7 @@ def main():
# Set options specific to the data generation job, then render the
# template.
pbs_options['job_name'] = 'genTestData'
pbs_options['walltime'] = '02:00:00'
pbs_options['walltime'] = '00:30:00'
pbs_content = data_generation_pbs_template.render(pbs_options)
if verbose:
print(f"Creating {DATA_GENERATION_PBS_SCRIPT}.")