diff --git a/testingScripts/combined_relative_tests_report.py b/testingScripts/combined_relative_tests_report.py new file mode 100755 index 00000000..a4380770 --- /dev/null +++ b/testingScripts/combined_relative_tests_report.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python + + +"""Create the combined MAGE relative test report. + +This script creates the combined MAGE relative test report for the serial and +MPI versions of the test. + +Authors +------- +Jeff Garretson +Eric Winter + +""" + + +# Import standard modules. +import copy +import datetime +import glob +import os +import platform +# import re +# import subprocess +import sys + +# Import 3rd-party modules. +# from astropy.time import Time +# import matplotlib as mpl +# import matplotlib.dates as mdates +# import matplotlib.pyplot as plt + +# Import project modules. +import common +# import kaipy.kaiH5 as kh5 +# import kaipy.kaiViz as kv + + +# Program constants + +# Program description. +DESCRIPTION = ( + "Create a combined report for the serial and MPI relative tests." +) + + +# Default values for command-line arguments. +DEFAULT_ARGUMENTS = { + "debug": False, + "loud": False, + "slack_on_fail": False, + "test": False, + "verbose": False, + "mpi1": "", + "mpi2": "", + "serial1": "", + "serial2": "", +} + +# # Root of directory tree for all tests. +# MAGE_TEST_ROOT = os.environ["MAGE_TEST_ROOT"] + +# Root of directory tree for this set of tests. +MAGE_TEST_SET_ROOT = os.environ["MAGE_TEST_SET_ROOT"] + +# Directory for relative tests +TEST_DIRECTORY = os.path.join(MAGE_TEST_SET_ROOT, "compTest") + +# Glob pattern for individual relative test directories. +COMP_TESTS_DIRECTORY_GLOB_PATTERN = "compTest_*" + +# # Regular expression for git hash read from weekly dash output log. +# GIT_HASH_PATTERN = "Git hash = (.{8})" + +# Name of subdirectory containing binaries and test results. +BIN_DIR = "bin" + +# # Name of file containg PBS job IDs. +# JOB_LIST_FILE = "jobs.txt" + +# String naming branch or commit used in this test. +BRANCH_OR_COMMIT = os.environ["BRANCH_OR_COMMIT"] + + +def create_command_line_parser(): + """Create the command-line parser. + + Create the parser for the command line. + + Parameters + ---------- + None + + Returns + ------- + parser : argparse.ArgumentParser + Command-line parser for this script. + + Raises + ------ + None + """ + parser = common.create_command_line_parser(DESCRIPTION) + parser.add_argument( + "mpi1", type=str, default=DEFAULT_ARGUMENTS["mpi1"], + help="Folder for first MPI case." + ) + parser.add_argument( + "mpi2", type=str, default=DEFAULT_ARGUMENTS["mpi2"], + help="Folder for second MPI case." + ) + parser.add_argument( + "serial1", type=str, default=DEFAULT_ARGUMENTS["serial1"], + help="Folder for first serial case." + ) + parser.add_argument( + "serial2", type=str, default=DEFAULT_ARGUMENTS["serial2"], + help="Folder for second serial case." + ) + return parser + + +def combined_relative_tests_report(args: dict): + """Create a combined test report for relativeTests.py. + + Create a combined test report for relativeTests.py. + + Parameters + ---------- + args: dict + Dictionary of command-line options. + + Returns + ------- + int 0 on success + + Raises + ------ + AssertionError + If an invalid argument is provided. + """ + # Set defaults for command-line options, then update with values passed + # from the caller. + local_args = copy.deepcopy(DEFAULT_ARGUMENTS) + local_args.update(args) + args = local_args + + # Local convenience variables. + debug = args["debug"] + loud = args["loud"] + slack_on_fail = args["slack_on_fail"] + test = args["test"] + verbose = args["verbose"] + mpi1 = args["mpi1"] + mpi2 = args["mpi2"] + serial1 = args["serial1"] + serial2 = args["serial2"] + + # Validate arguments. + assert len(serial1) > 0 + assert len(serial2) > 0 + assert len(mpi1) > 0 + assert len(mpi2) > 0 + + # ------------------------------------------------------------------------ + +# # Add additional arguments +# parser.add_argument( +# "-d1", type=str, help="Folder for the first case" +# ) +# parser.add_argument( +# "-d2", type=str, help="Folder for the second case" +# ) +# parser.add_argument( +# "-cn", type=str, help="Name of this post process case" +# ) + +# # Parse the command-line arguments. +# args = parser.parse_args() +# if args.debug: +# print(f"args = {args}") +# debug = args.debug +# be_loud = args.loud +# # slack_on_fail = args.slack_on_fail +# is_test = args.test +# verbose = args.verbose +# d1 = args.d1 +# d2 = args.d2 +# cn = args.cn + + # ------------------------------------------------------------------------ + + if debug: + print(f"Starting {sys.argv[0]} at {datetime.datetime.now()}" + f" on {platform.node()}") + print(f"Current directory is {os.getcwd()}") + + # ------------------------------------------------------------------------ + + # Move to the top-level weekly dash directory. + os.chdir(TEST_DIRECTORY) + + # ------------------------------------------------------------------------ + + # Get list of comparison test directories. + test_directories = glob.glob(COMP_TESTS_DIRECTORY_GLOB_PATTERN) + if debug: + print(f"comp_test_directories = {test_directories}") + + # + # Use only the first directory for now. + test_directory = test_directories[0] + # + + # ------------------------------------------------------------------------ + + # Read results from the latest run. + if verbose: + print(f"Reading results for latest run in {test_directory}.") + + # Go to test folder + os.chdir(test_directory) + + # Move down to the build directory + os.chdir(BIN_DIR) + + # ----------------------------------------------------------------------- + + # Detail the test results. + test_report_details_string = "" + test_report_details_string += ( + f"Test results are on `derecho` in `{test_directory}`.\n" + ) + + # Summarize the test results. + test_report_summary_string = ( + "Comparative test result plots complete on branch " + f"`{BRANCH_OR_COMMIT}`." + ) + + # Print the test results summary and details. + print(test_report_summary_string) + print(test_report_details_string) + + # If loud mode is on, post results to Slack. + if loud: + slack_client = common.slack_create_client() + if debug: + print(f"slack_client = {slack_client}") + slack_response = common.slack_send_message( + slack_client, test_report_summary_string, is_test=test + ) + if slack_response["ok"]: + parent_ts = slack_response["ts"] + message = ( + "All results are from a Double Resolution Run using" + " various Gamera and Voltron settings.\n" + ) + slack_response = common.slack_send_message( + slack_client, message, thread_ts=parent_ts, is_test=test + ) + message = f"This result compared `{mpi1}` and `{mpi2}`.\n" + slack_response = common.slack_send_image( + slack_client, os.path.join("vidData", "MpiRestartComp.mp4"), + initial_comment=message, + thread_ts=parent_ts, is_test=test + ) + message = f"This result compared `{serial1}` and `{serial2}`.\n" + slack_response = common.slack_send_image( + slack_client, os.path.join("vidData", "SerialMpiComp.mp4"), + initial_comment=message, + thread_ts=parent_ts, is_test=test + ) + else: + print("Failed to post parent message and images to Slack.") + + # ------------------------------------------------------------------------ + + if debug: + print(f"Ending {sys.argv[0]} at {datetime.datetime.now()}" + f" on {platform.node()}") + + # Return normally. + return 0 + + +def main(): + """Driver for command-line version of code.""" + # Set up the command-line parser. + parser = create_command_line_parser() + + # Parse the command-line arguments. + args = parser.parse_args() + if args.debug: + print(f"args = {args}") + + # Convert the arguments from Namespace to dict. + args = vars(args) + + # Call the main program code. + return_code = combined_relative_tests_report(args) + sys.exit(return_code) + + +if __name__ == "__main__": + main() diff --git a/testingScripts/relCasePost-template.pbs b/testingScripts/relCasePost-template.pbs index 5a5be109..2f968271 100644 --- a/testingScripts/relCasePost-template.pbs +++ b/testingScripts/relCasePost-template.pbs @@ -50,8 +50,5 @@ printenv # Process the data and generate the output video python $KAIPYHOME/kaipy/scripts/quicklook/gamerrVid.py -d1 {{ case1F }} -id1 {{ case1id }} -d2 {{ case2F }} -id2 {{ case2id }} -o {{ frameFolder }}/{{ caseName }} -ts {{ ts }} -te {{ te }} -dt {{ dt }} -Nth 9 >& {{ caseName }}.out -# Generate the report. -python $KAIJUHOME/testingScripts/relativeTestsReport.py {{ report_options }} -d1 {{ case1F }} -d2 {{ case2F }} -cn {{ caseName }} >& {{ job_name }}_report.out - echo "Job $PBS_JOBID ended at `date` on `hostname` in directory `pwd`." diff --git a/testingScripts/relCaseReport-template.pbs b/testingScripts/relCaseReport-template.pbs new file mode 100644 index 00000000..1695affb --- /dev/null +++ b/testingScripts/relCaseReport-template.pbs @@ -0,0 +1,52 @@ +#!/bin/bash + +#PBS -N relCaseReport +#PBS -A {{ account }} +#PBS -q casper@casper-pbs +#PBS -l job_priority={{ job_priority }} +#PBS -l select=1:ncpus=128 +#PBS -l walltime=1:00:00 +#PBS -j oe +#PBS -m abe + +echo "Job $PBS_JOBID started at `date` on `hostname` in directory `pwd`." + +echo 'Loading python environment.' +mage_test_root='{{ mage_test_root }}' +export CONDARC="${mage_test_root}/.condarc" +export CONDA_ENVS_PATH="${mage_test_root}/.conda" +mage_miniconda3="${mage_test_root}/miniconda3" +mage_conda="${mage_miniconda3}/bin/conda" +__conda_setup="$($mage_conda 'shell.bash' 'hook' 2> /dev/null)" +if [ $? -eq 0 ]; then + eval "$__conda_setup" +else + if [ -f "$mage_miniconda3/etc/profile.d/conda.sh" ]; then + . "$mage_miniconda3/etc/profile.d/conda.sh" + else + export PATH="$mage_miniconda3/bin:$PATH" + fi +fi +unset __conda_setup +conda activate kaiju-3.8-testing + +#echo 'Setting up MAGE environment.' +source {{ kaijuhome }}/scripts/setupEnvironment.sh +source {{ kaipyhome }}/kaipy/scripts/setupEnvironment.sh + +echo 'Setting environment variables.' +export TMPDIR={{ tmpdir }} +export SLACK_BOT_TOKEN={{ slack_bot_token }} +export KMP_STACKSIZE=128M +export MAGE_TEST_ROOT=$mage_test_root +export MAGE_TEST_SET_ROOT={{ mage_test_set_root }} +export BRANCH_OR_COMMIT={{ branch_or_commit }} +export JOBNAME={{ job_name }} +echo 'The active environment variables are:' +printenv + +# Generate the report. +$KAIJUHOME/testingScripts/combined_relative_tests_report.py -ltv relMpi32ResRelease relMpi32Release relSerialRelease relMpi32Release + +echo "Job $PBS_JOBID ended at `date` on `hostname` in directory `pwd`." + diff --git a/testingScripts/relativeTests.py b/testingScripts/relativeTests.py index 40555f2a..649f23cd 100755 --- a/testingScripts/relativeTests.py +++ b/testingScripts/relativeTests.py @@ -72,6 +72,11 @@ POSTPROC_TEMPLATE = os.path.join( TEST_SCRIPTS_DIRECTORY, 'relCasePost-template.pbs' ) +# Path to jinja2 template file for combined Slack report PBS script. +SLACK_REPORT_TEMPLATE = os.path.join( + TEST_SCRIPTS_DIRECTORY, 'relCaseReport-template.pbs' +) + # Path to jinja2 template file for XML file XML_TEMPLATE = os.path.join( @@ -109,6 +114,34 @@ def processComparativeResults(case1_jobid, case2_jobid, caseName, pbsTemplate, p job_id = cproc.stdout.split('.')[0] if debug: print(f"job_id = {job_id}") + return job_id + + +def create_combined_report(postproc_job_id, caseName, pbsTemplate, pbs_options): + # Render the job template. + pbs_content = pbsTemplate.render(pbs_options) + pbsFilename = f"{caseName}.pbs" + with open(pbsFilename, 'w', encoding='utf-8') as f: + f.write(pbs_content) + # Submit the job + if verbose: + print('Submitting combined Slack report job.') + cmd = f"qsub -W depend=afterok:{postproc_job_id} {pbsFilename}" + if debug: + print(f"cmd = {cmd}") + try: + cproc = subprocess.run(cmd, shell=True, check=True, + text=True, capture_output=True) + except subprocess.CalledProcessError as e: + print('ERROR: qsub failed.\n' + f"e.cmd = {e.cmd}\n" + f"e.returncode = {e.returncode}\n" + 'See test log for output.\n', + file=sys.stderr) + job_id = cproc.stdout.split('.')[0] + if debug: + print(f"job_id = {job_id}") + def generateAndRunCase(caseName,pbsTemplate,pbs_options,xmlTemplate,xml_options,wait_job_id=None): os.mkdir(caseName) @@ -435,6 +468,15 @@ def main(): # ------------------------------------------------------------------------ + # Read the template for the PBS script used for posting reports to Slack. + with open(SLACK_REPORT_TEMPLATE, 'r', encoding='utf-8') as f: + template_content = f.read() + slack_report_template = Template(template_content) + if debug: + print(f"slack_report_template = {slack_report_template}") + + # ------------------------------------------------------------------------ + # Read the template for the XML file used for the test data generation. with open(XML_TEMPLATE, 'r', encoding='utf-8') as f: @@ -679,20 +721,12 @@ def main(): postProcOpts['ts'] = '0' postProcOpts['te'] = '120' postProcOpts['dt'] = '60' - processComparativeResults(job_id_s_r, job_id_m32_r, postProcOpts['caseName'], postproc_template, postProcOpts) - + postproc_job_id = processComparativeResults(job_id_s_r, job_id_m32_r, postProcOpts['caseName'], postproc_template, postProcOpts) + + # Generate the combined report. postProcOpts = base_pbs_options - postProcOpts['caseName'] = 'MpiRestartComp' - postProcOpts['frameFolder'] = 'vidData' - postProcOpts['case1F'] = 'relMpi32Release' - postProcOpts['case1id'] = 'msphere_M32_R' - postProcOpts['case2F'] = 'relMpi32ResRelease' - postProcOpts['case2id'] = 'msphere_M32_R' - postProcOpts['ts'] = '50' - postProcOpts['te'] = '120' - postProcOpts['dt'] = '60' - processComparativeResults(job_id_m32_r, job_id_m32r_r, postProcOpts['caseName'], postproc_template, postProcOpts) - + create_combined_report(postproc_job_id, "combined_report", slack_report_template, base_pbs_options) + else: if debug: print("Performing full test suite") diff --git a/testingScripts/relativeTestsReport.py b/testingScripts/relativeTestsReport.py index 843c7ab1..3ba87552 100755 --- a/testingScripts/relativeTestsReport.py +++ b/testingScripts/relativeTestsReport.py @@ -157,13 +157,13 @@ def main(): 'Results attached as replies to this ' 'message.\n' ) - message += ( - f"Test results are in {os.getcwd()}.\n" - ) slack_response = common.slack_send_message( slack_client, message, is_test=is_test) if slack_response['ok']: parent_ts = slack_response['ts'] + message += ( + f"Test results are in {os.getcwd()}.\n" + ) message = ( 'All results are from a Double Resolution Run using' ' various Gamera and Voltron settings.'