Merged in ewinter-derecho_testing (pull request #47)

Roll-up merge of a few months of testing development Approved-by: Slava Merkin Approved-by: Jeff
2026-01-08 22:58:05 -05:00 · 2025-10-06 21:48:28 +00:00
parent 4a71455cfe 3cfe636d57
commit 0671888895
21 changed files with 473 additions and 395 deletions
--- a/testingScripts/compare_mage_runs.py
+++ b/testingScripts/compare_mage_runs.py
@@ -384,7 +384,7 @@ def create_magnetosphere_quicklook_plots(xml_files: list):
        os.chdir(results_dir)

        # Create the quicklook plot.
-        cmd = f"msphpic.py -id {runid}"
+        cmd = f"msphpic -id {runid}"
        _ = subprocess.run(cmd, shell=True, check=True)

        # Move back to the starting directory.
@@ -464,7 +464,7 @@ def create_REMIX_quicklook_plots(xml_files: list):
        os.chdir(results_dir)

        # Create the quicklook plots.
-        cmd = f"mixpic.py -id {runid}"
+        cmd = f"mixpic -id {runid}"
        _ = subprocess.run(cmd, shell=True, check=True)

        # Add the plots to the lists.
@@ -517,83 +517,6 @@ def merge_REMIX_quicklook_plots(plots_north: list, plots_south: list):
    return merged_plot_north, merged_plot_south


-def create_RCM_quicklook_plots(xml_files: list):
-    """Create the RCM quicklook plot for each run.
-
-    Create the RCM quicklook plot for each run.
-
-    Parameters
-    ----------
-    xml_files : list of str
-        List of XML files describing each run.
-
-    Returns
-    -------
-    quicklook_plots : list of str
-        Path to each quicklook file.
-
-    Raises
-    ------
-    None
-    """
-    # Save the current directory.
-    cwd = os.getcwd()
-
-    # Make the RCM quicklook plot for each run.
-    quicklook_plots = []
-    for xml_file in xml_files:
-
-        # Extract the run ID.
-        runid = common.extract_runid(xml_file)
-
-        # Compute the path to the results directory.
-        results_dir = os.path.split(xml_file)[0]
-
-        # Move to the results directory.
-        os.chdir(results_dir)
-
-        # Create the quicklook plot.
-        cmd = f"rcmpic.py -id {runid}"
-        _ = subprocess.run(cmd, shell=True, check=True)
-
-        # Add the plot to the list.
-        path = os.path.join(results_dir, "qkrcmpic.png")
-        quicklook_plots.append(path)
-
-        # Return to the original directory.
-        os.chdir(cwd)
-
-    # Return the list of quicklook plots.
-    return quicklook_plots
-
-
-def merge_RCM_quicklook_plots(quicklook_plots: list):
-    """Merge the RCM quicklook plots for all runs.
-
-    Merge the RCM quicklook plots for all runs.
-
-    Parameters
-    ----------
-    quicklook_plots : list of str
-        List of quicklook plots to merge.
-
-    Returns
-    -------
-    merged_plot : str
-        Path to merged quicklook file.
-
-    Raises
-    ------
-    None
-    """
-    # Merge RCM quicklooks.
-    merged_plot = "combined_qkrcmpic.png"
-    cmd = f"convert {' '.join(quicklook_plots)} -append {merged_plot}"
-    print(f"cmd = {cmd}")
-    _ = subprocess.run(cmd, shell=True, check=True)
-    return merged_plot
-
-
 def compare_mage_runs(args):
    """Compare a set of MAGE model runs.

@@ -710,20 +633,6 @@ def compare_mage_runs(args):
        print(f"merged_remix_plot_n = {merged_remix_plot_n}")
        print(f"merged_remix_plot_s = {merged_remix_plot_s}")

-    # Create the RCM quicklook plots.
-    if verbose:
-        print("Creating RCM quicklook plots.")
-    rcm_plots = create_RCM_quicklook_plots(run_xml_files)
-    if debug:
-        print(f"rcm_plots = {rcm_plots}")
-
-    # Create the merged RCM quicklook plots.
-    if verbose:
-        print("Creating merged RCM quicklook plot.")
-    rcm_merged_plots = merge_RCM_quicklook_plots(rcm_plots)
-    if debug:
-        print(f"rcm_merged_plots = {rcm_merged_plots}")
-
    # ------------------------------------------------------------------------

    # Post images to Slack.
@@ -736,7 +645,6 @@ def compare_mage_runs(args):
        "combined_msphpic.png",
        "combined_remix_n.png",
        "combined_remix_s.png",
-        "combined_qkrcmpic.png",
    ]
    comments_to_post = [
        "Real-Time Performance\n\n",
@@ -745,7 +653,6 @@ def compare_mage_runs(args):
        "Magnetosphere Quicklook Comparison Plots\n\n",
        "REMIX (north) Quicklook Comparison Plots\n\n",
        "REMIX (south) Quicklook Comparison Plots\n\n",
-        "RCM Quicklook Comparison Plots\n\n",
    ]

    # If loud mode is on, post results to Slack.
--- a/testingScripts/compare_mage_runs_numerical.py
+++ b/testingScripts/compare_mage_runs_numerical.py
@@ -125,67 +125,6 @@ def compare_GAMERA_results(runxml1: str, runxml2: str, verbose: bool = False):
    return TEST_PASS


-def compare_MHDRCM_results(runxml1: str, runxml2: str, verbose: bool = False):
-    """Numerically compare the MHD RCM output files from two runs.
-
-    Numerically compare the MHD RCM output files from two runs.
-
-    Parameters
-    ----------
-    runxm1 : str
-        Path to XML file describing 1st run.
-    runxm2 : str
-        Path to XML file describing 2nd run.
-    verbose : bool
-        Set to True to print verbose information during comparison.
-
-    Returns
-    -------
-    TEST_PASS or TEST_FAIL : str
-        Description of result of comparison.
-
-    Raises
-    ------
-    None
-    """
-    # Determine the directories containing the sets of results.
-    dir1 = os.path.split(runxml1)[0]
-    dir2 = os.path.split(runxml2)[0]
-
-    # Generate a sorted list of output files for the 1st run.
-    pattern1 = os.path.join(dir1, "*.mhdrcm.h5")
-    files1 = glob.glob(pattern1)
-    files = [os.path.split(f)[1] for f in files1]
-    files.sort()
-
-    # Compare each output file in the two directories.
-    # Comparisons are done with h5diff, which must be in the PATH.
-    # Attributes of the steps and other top-level groups are excluded from
-    # comparison.
-    for filename in files:
-        file1 = os.path.join(dir1, filename)
-        file2 = os.path.join(dir2, filename)
-        if verbose:
-            print(f"Numerically comparing {file1} to {file2}.")
-
-        # Compare each step, without attributes.
-        _, step_ids = kaiH5.cntSteps(file1)
-        for step_id in step_ids:
-            step_path = f"/Step#{step_id}"
-            if verbose:
-                print(f"  Comparing {step_path}.")
-            cmd = (
-                f"h5diff --exclude-attribute {step_path} {file1} {file2} "
-                f"{step_path}"
-            )
-            cproc = subprocess.run(cmd, shell=True, check=True)
-            if cproc.returncode != 0:
-                return TEST_FAIL
-
-    # Return the result of the comparison.
-    return TEST_PASS
-
-
 def compare_REMIX_results(runxml1: str, runxml2: str, verbose: bool = False):
    """Numerically compare the REMIX output files from two runs.

@@ -247,67 +186,6 @@ def compare_REMIX_results(runxml1: str, runxml2: str, verbose: bool = False):
    return TEST_PASS


-def compare_RCM_results(runxml1: str, runxml2: str, verbose: bool = False):
-    """Numerically compare the RCM output files from two runs.
-
-    Numerically compare the RCM output files from two runs.
-
-    Parameters
-    ----------
-    runxm1 : str
-        Path to XML file describing 1st run.
-    runxm2 : str
-        Path to XML file describing 2nd run.
-    verbose : bool
-        Set to True to print verbose information during comparison.
-
-    Returns
-    -------
-    TEST_PASS or TEST_FAIL : str
-        Description of result of comparison.
-
-    Raises
-    ------
-    None
-    """
-    # Determine the directories containing the sets of results.
-    dir1 = os.path.split(runxml1)[0]
-    dir2 = os.path.split(runxml2)[0]
-
-    # Generate a sorted list of output files for the 1st run.
-    pattern1 = os.path.join(dir1, "*.rcm.h5")
-    files1 = glob.glob(pattern1)
-    files = [os.path.split(f)[1] for f in files1]
-    files.sort()
-
-    # Compare each result file in the two directories.
-    # Comparisons are done with h5diff, which must be in the PATH.
-    # Attributes of the steps and other top-level groups are excluded from
-    # comparison.
-    for filename in files:
-        file1 = os.path.join(dir1, filename)
-        file2 = os.path.join(dir2, filename)
-        if verbose:
-            print(f"Numerically comparing {file1} to {file2}.")
-
-        # Compare each step, without attributes.
-        _, step_ids = kaiH5.cntSteps(file1)
-        for step_id in step_ids:
-            step_path = f"/Step#{step_id}"
-            if verbose:
-                print(f"  Comparing {step_path}.")
-            cmd = (
-                f"h5diff --exclude-attribute {step_path} {file1} {file2} "
-                f"{step_path}"
-            )
-            cproc = subprocess.run(cmd, shell=True, check=True)
-            if cproc.returncode != 0:
-                return TEST_FAIL
-
-    # Return the result of the comparison.
-    return TEST_PASS
-
-
 def compare_VOLTRON_results(runxml1: str, runxml2: str, verbose: bool = False):
    """Numerically compare the VOLTRON output files from two runs.

@@ -417,14 +295,6 @@ def compare_mage_runs_numerical(args: dict):
        print(f"comparison_result = {comparison_result}")
    comparison_results.append(comparison_result)

-    # Compare the MHD RCM output files.
-    if verbose:
-        print("Comparing MHD RCM output files.")
-    comparison_result = compare_MHDRCM_results(runxml1, runxml2, verbose)
-    if debug:
-        print(f"comparison_result = {comparison_result}")
-    comparison_results.append(comparison_result)
-
    # Compare the REMIX output files.
    if verbose:
        print("Comparing REMIX output files.")
@@ -433,14 +303,6 @@ def compare_mage_runs_numerical(args: dict):
        print(f"comparison_result = {comparison_result}")
    comparison_results.append(comparison_result)

-    # Compare the RCM output files.
-    if verbose:
-        print("Comparing RCM output files.")
-    comparison_result = compare_RCM_results(runxml1, runxml2, verbose)
-    if debug:
-        print(f"comparison_result = {comparison_result}")
-    comparison_results.append(comparison_result)
-
    # Compare the VOLTRON output files.
    if verbose:
        print("Comparing VOLTRON output files.")
@@ -454,10 +316,8 @@ def compare_mage_runs_numerical(args: dict):
    # Detail the test results.
    test_report_details_string = ""
    test_report_details_string += f"GAMERA: *{comparison_results[0]}*\n"
-    test_report_details_string += f"MHD RCM: *{comparison_results[1]}*\n"
-    test_report_details_string += f"REMIX: *{comparison_results[2]}*\n"
-    test_report_details_string += f"RCM: *{comparison_results[3]}*\n"
-    test_report_details_string += f"VOLTRON: *{comparison_results[4]}*\n"
+    test_report_details_string += f"REMIX: *{comparison_results[1]}*\n"
+    test_report_details_string += f"VOLTRON: *{comparison_results[2]}*\n"

    # Summarize the test results.
    if run_description is not None:
--- a/testingScripts/crontab
+++ b/testingScripts/crontab
@@ -16,8 +16,7 @@
 # source tree). Job submission reports to Slack only on failure, report run
 # in Slack-on-fail (-s) mode.
 15 00 * * * ssh derecho "/glade/campaign/hao/msphere/automated_kaiju_tests/kaiju-private/testingScripts/run_mage_tests.sh -v -b development 'unitTest.py -sv'" >> /glade/campaign/hao/msphere/automated_kaiju_tests/logs/nightly-tests-2-development.out 2>&1
-# Disable master unit test for now since it hangs.
-# 20 00 * * * ssh derecho "/glade/campaign/hao/msphere/automated_kaiju_tests/kaiju-private/testingScripts/run_mage_tests.sh -v -b master      'unitTest.py -sv'" >> /glade/campaign/hao/msphere/automated_kaiju_tests/logs/nightly-tests-2-master.out      2>&1
+20 00 * * * ssh derecho "/glade/campaign/hao/msphere/automated_kaiju_tests/kaiju-private/testingScripts/run_mage_tests.sh -v -b master      'unitTest.py -sv'" >> /glade/campaign/hao/msphere/automated_kaiju_tests/logs/nightly-tests-2-master.out      2>&1

 # Run a weekly dash every Sunday morning for the development branch and
 # compare to reference development run listed in reference_runs.txt.
--- a/testingScripts/derecho_configuration_check-template.pbs
+++ b/testingScripts/derecho_configuration_check-template.pbs
@@ -20,21 +20,18 @@ echo 'The currently loaded modules are:'
 module list

 echo 'Loading python environment.'
-export CONDARC="{{ condarc }}"
-export CONDA_ENVS_PATH="{{ conda_envs_path }}"
-mage_miniconda3="{{ mage_test_root }}/miniconda3"
-mage_conda="${mage_miniconda3}/bin/conda"
-__conda_setup="$($mage_conda 'shell.bash' 'hook' 2> /dev/null)"
+__conda_setup="$('/glade/u/home/ewinter/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)"
 if [ $? -eq 0 ]; then
    eval "$__conda_setup"
 else
-    if [ -f "$mage_miniconda3/etc/profile.d/conda.sh" ]; then
-        . "$mage_miniconda3/etc/profile.d/conda.sh"
+    if [ -f "/glade/u/home/ewinter/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/glade/u/home/ewinter/miniconda3/etc/profile.d/conda.sh"
    else
-        export PATH="$mage_miniconda3/bin:$PATH"
+        export PATH="/glade/u/home/ewinter/miniconda3/bin:$PATH"
    fi
 fi
 unset __conda_setup
+
 conda activate {{ conda_environment }}
 echo "The current conda environment is ${CONDA_PREFIX}."

@@ -42,10 +39,6 @@ echo 'Setting up MAGE environment.'
 source {{ kaijuhome }}/scripts/setupEnvironment.sh
 echo "The kaiju software is located at ${KAIJUHOME}."

-echo 'Setting up kaipy environment.'
-source {{ kaipy_private_root }}/kaipy/scripts/setupEnvironment.sh
-echo "The kaipy software is located at ${KAIPYHOME}."
-
 echo 'Setting environment variables.'
 export TMPDIR={{ tmpdir }}
 export SLACK_BOT_TOKEN={{ slack_bot_token }}
@@ -82,10 +75,10 @@ echo 'Generating the solar wind boundary condition file.'
 {{ cda2wind_cmd }}
 echo "The solar wind boundary condition file is `ls bcwind.h5`."

-# Generate the RCM configuration file.
-echo 'Generating the RCM configuration file.'
-{{ genRCM_cmd }}
-echo "The RCM configuration file is `ls rcmconfig.h5`."
+# Generate the RAIJU configuration file.
+echo 'Generating the RAIJU configuration file.'
+genRAIJU
+echo "The RAIJU configuration file is `ls raijuconfig.h5`."

 # Moved this setting here to avoid error from genLFM.py.
 export OMP_NUM_THREADS=128
@@ -99,7 +92,7 @@ eval $cmd

 # Run the comparison. Post results to Slack if any test fails.
 reference_run=`cat /glade/u/home/ewinter/mage_testing/test_runs/derecho_configuration_check_runs.txt`
-cmd="python $KAIJUHOME/testingScripts/compare_mage_runs_numerical.py -sv --run_description='`derecho` configuration check' $reference_run `pwd`/weeklyDashGo.xml >& compare_mage_runs_numerical.out"
+cmd="python $KAIJUHOME/testingScripts/compare_mage_runs_numerical.py -sv --run_description='derecho configuration check' $reference_run `pwd`/weeklyDashGo.xml >& compare_mage_runs_numerical.out"
 echo "Run comparison command is:"
 echo $cmd
 eval $cmd
--- a/testingScripts/derecho_configuration_check.py
+++ b/testingScripts/derecho_configuration_check.py
@@ -143,16 +143,13 @@ def derecho_configuration_check(args: dict):
    make_cmd = "make voltron_mpi.x"

    # Create the command to generate the LFM grid.
-    genLFM_cmd = "genLFM.py -gid Q"
+    genLFM_cmd = "genLFM -gid Q"

    # Create the command to generate the solar wind boundary condition file.
    cda2wind_cmd = (
-        "cda2wind.py -t0 2016-08-09T02:00:00 -t1 2016-08-09T12:00:00"
+        "cda2wind -t0 2016-08-09T02:00:00 -t1 2016-08-09T12:00:00"
    )

-    # Create the command to generate the RCM configuration.
-    genRCM_cmd = "genRCM.py"
-
    # Create the command for launching an MPI program.
    mpiexec_cmd = f"mpiexec {KAIJUHOME}/scripts/preproc/pinCpuCores.sh"

@@ -179,13 +176,10 @@ def derecho_configuration_check(args: dict):
    pbs_options["job_priority"] = os.environ["DERECHO_TESTING_PRIORITY"]
    pbs_options["walltime"] = "08:00:00"
    pbs_options["modules"] = module_names
-    pbs_options["condarc"] = os.environ["CONDARC"]
-    pbs_options["conda_envs_path"] = os.environ["CONDA_ENVS_PATH"]
    pbs_options["conda_environment"] = os.environ["CONDA_ENVIRONMENT"]
    pbs_options["mage_test_root"] = os.environ["MAGE_TEST_ROOT"]
    pbs_options["mage_test_set_root"] = os.environ["MAGE_TEST_SET_ROOT"]
    pbs_options["kaijuhome"] = KAIJUHOME
-    pbs_options["kaipy_private_root"] = os.environ["KAIPY_PRIVATE_ROOT"]
    pbs_options["tmpdir"] = os.environ["TMPDIR"]
    pbs_options["slack_bot_token"] = os.environ["SLACK_BOT_TOKEN"]
    pbs_options["branch_or_commit"] = os.environ["BRANCH_OR_COMMIT"]
@@ -193,7 +187,6 @@ def derecho_configuration_check(args: dict):
    pbs_options["make_cmd"] = make_cmd
    pbs_options["genLFM_cmd"] = genLFM_cmd
    pbs_options["cda2wind_cmd"] = cda2wind_cmd
-    pbs_options["genRCM_cmd"] = genRCM_cmd
    pbs_options["mpiexec_cmd"] = mpiexec_cmd
    pbs_options["voltron_cmd"] = voltron_cmd

--- a/testingScripts/intelCheckSubmitReport-template.pbs
+++ b/testingScripts/intelCheckSubmitReport-template.pbs
@@ -19,11 +19,9 @@ module load {{ module }}
 module list

 echo 'Loading python environment.'
+mage_test_root=$HOME
 if [ -d "${mage_test_root}/miniconda3" ]; then
    echo 'Loading local miniconda3'
-    mage_test_root='{{ mage_test_root }}'
-    export CONDARC="${mage_test_root}/.condarc"
-    export CONDA_ENVS_PATH="${mage_test_root}/.conda"
    mage_miniconda3="${mage_test_root}/miniconda3"
    mage_conda="${mage_miniconda3}/bin/conda"
    __conda_setup="$($mage_conda 'shell.bash' 'hook' 2> /dev/null)"
@@ -45,7 +43,6 @@ conda activate {{ conda_environment }}

 echo 'Setting up MAGE environment.'
 source {{ kaijuhome }}/scripts/setupEnvironment.sh
-source {{ kaipyhome }}/kaipy/scripts/setupEnvironment.sh

 echo 'Setting environment variables.'
 export TMPDIR={{ tmpdir }}
--- a/testingScripts/intelChecks.py
+++ b/testingScripts/intelChecks.py
@@ -305,7 +305,7 @@ def intelChecks(args: dict):
        # Generate bcwind data file.
        if verbose:
            print("Creating bcwind data file.")
-        cmd = "cda2wind.py -t0 2016-08-09T09:00:00 -t1 2016-08-09T11:00:00"
+        cmd = "cda2wind -t0 2016-08-09T09:00:00 -t1 2016-08-09T11:00:00"
        if debug:
            print(f"cmd = {cmd}")
        try:
@@ -315,7 +315,7 @@ def intelChecks(args: dict):
                  f"{module_set_name}.\n"
                  f"e.cmd = {e.cmd}\n"
                  f"e.returncode = {e.returncode}\n"
-                  "See testing log for output from cda2wind.py.\n"
+                  "See testing log for output from cda2wind.\n"
                  "Skipping remaining steps for module set"
                  f"{module_set_name}\n")
            continue
@@ -325,7 +325,7 @@ def intelChecks(args: dict):
        # Generate the LFM grid file.
        if verbose:
            print("Creating LFM grid file.")
-        cmd = "genLFM.py -gid D"
+        cmd = "genLFM -gid D"
        if debug:
            print(f"cmd = {cmd}")
        try:
@@ -335,7 +335,7 @@ def intelChecks(args: dict):
                  f"{module_set_name}.\n"
                  f"e.cmd = {e.cmd}\n"
                  f"e.returncode = {e.returncode}\n"
-                  "See testing log for output from genLFM.py.\n"
+                  "See testing log for output from genLFM.\n"
                  "Skipping remaining steps for module set"
                  f"{module_set_name}\n")
            continue
@@ -345,7 +345,7 @@ def intelChecks(args: dict):
        # Generate the Raiju configuration file.
        if verbose:
            print("Creating Raiju configuration file.")
-        cmd = "genRAIJU.py"
+        cmd = "genRAIJU"
        if debug:
            print(f"cmd = {cmd}")
        try:
@@ -355,7 +355,7 @@ def intelChecks(args: dict):
                  f" for module set {module_set_name}.\n"
                  f"e.cmd = {e.cmd}\n"
                  f"e.returncode = {e.returncode}\n"
-                  "See testing log for output from genRAIJU.py.\n"
+                  "See testing log for output from genRAIJU.\n"
                  "Skipping remaining steps for module set "
                  f"{module_set_name}\n")
            continue
@@ -373,7 +373,6 @@ def intelChecks(args: dict):
        pbs_options["job_priority"] = os.environ["DERECHO_TESTING_PRIORITY"]
        pbs_options["modules"] = module_names
        pbs_options["kaijuhome"] = KAIJUHOME
-        pbs_options["kaipyhome"] = os.environ["KAIPYHOME"]
        pbs_options["tmpdir"] = os.environ["TMPDIR"]
        pbs_options["slack_bot_token"] = os.environ["SLACK_BOT_TOKEN"]
        pbs_options["mage_test_root"] = os.environ["MAGE_TEST_ROOT"]
--- a/testingScripts/mage_reproducibility_check-template.pbs
+++ b/testingScripts/mage_reproducibility_check-template.pbs
@@ -19,11 +19,11 @@ module load {{ module }}
 echo 'The currently loaded modules are:'
 module list

+echo 'Loading python environment.'
+mage_test_root=$HOME
 if [ -d "${mage_test_root}/miniconda3" ]; then
    echo 'Loading local miniconda3'
-    export CONDARC="{{ condarc }}"
-    export CONDA_ENVS_PATH="{{ conda_envs_path }}"
-    mage_miniconda3="{{ mage_test_root }}/miniconda3"
+    mage_miniconda3="${mage_test_root}/miniconda3"
    mage_conda="${mage_miniconda3}/bin/conda"
    __conda_setup="$($mage_conda 'shell.bash' 'hook' 2> /dev/null)"
    if [ $? -eq 0 ]; then
@@ -47,10 +47,6 @@ echo 'Setting up MAGE environment.'
 source {{ kaijuhome }}/scripts/setupEnvironment.sh
 echo "The kaiju software is located at ${KAIJUHOME}."

-echo 'Setting up kaipy environment.'
-source {{ kaipy_private_root }}/kaipy/scripts/setupEnvironment.sh
-echo "The kaipy software is located at ${KAIPYHOME}."
-
 echo 'Setting environment variables.'
 export TMPDIR={{ tmpdir }}
 export SLACK_BOT_TOKEN={{ slack_bot_token }}
--- a/testingScripts/mage_reproducibility_check.py
+++ b/testingScripts/mage_reproducibility_check.py
@@ -84,6 +84,35 @@ MAGE_REPRODUCIBILITY_CHECK_PBS_TEMPLATE_FILE = os.path.join(
 MAGE_REPRODUCIBILITY_CHECK_PBS_SCRIPT = "mage_reproducibility_check.pbs"


+def create_command_line_parser():
+    """Create the command-line argument parser.
+
+    Create the parser for command-line arguments.
+
+    Parameters
+    ----------
+    None
+
+    Returns
+    -------
+    parser : argparse.ArgumentParser
+        Command-line argument parser for this script.
+
+    Raises
+    ------
+    None
+    """
+    parser = common.create_command_line_parser(DESCRIPTION)
+    parser.add_argument(
+        "--module_set_file", "-f", default=DEFAULT_MODULE_SET_FILE,
+        help=(
+            "Path to text file containing set of modules to build with "
+            "(default: %(default)s)"
+        )
+    )
+    return parser
+
+
 def mage_reproducibility_check(args: dict):
    """Perform a MAGE reproducibility check.

@@ -146,7 +175,6 @@ def mage_reproducibility_check(args: dict):
    module_names, cmake_environment, cmake_options = (
        common.read_build_module_list_file(module_set_file)
    )
-    print(f"module_names = {module_names}")

    # Extract the name of the list.
    module_set_name = os.path.split(module_set_file)[-1].rstrip(".lst")
@@ -213,12 +241,11 @@ def mage_reproducibility_check(args: dict):
    pbs_template = Template(template_content)

    # Assemble commands needed in the PBS script.
-    genLFM_cmd = "genLFM.py -gid Q"
+    genLFM_cmd = "genLFM -gid Q"
    cda2wind_cmd = (
-        "cda2wind.py -t0 2016-08-09T02:00:00 -t1 2016-08-09T12:00:00"
+        "cda2wind -t0 2016-08-09T02:00:00 -t1 2016-08-09T12:00:00"
    )
-    genRCM_cmd = "genRCM.py"
-    genRaiju_cmd = "genRAIJU.py"
+    genRaiju_cmd = "genRAIJU"
    mpiexec_cmd = f"mpiexec {KAIJUHOME}/scripts/preproc/pinCpuCores.sh"
    voltron_cmd = "../bin/voltron_mpi.x weeklyDashGo.xml"

@@ -241,13 +268,10 @@ def mage_reproducibility_check(args: dict):
    pbs_options["job_priority"] = os.environ["DERECHO_TESTING_PRIORITY"]
    pbs_options["walltime"] = "08:00:00"
    pbs_options["modules"] = module_names
-    pbs_options["condarc"] = os.environ["CONDARC"]
-    pbs_options["conda_envs_path"] = os.environ["CONDA_ENVS_PATH"]
-    pbs_options["conda_environment"] = os.environ["CONDA_ENVIRONMENT"]
    pbs_options["mage_test_root"] = os.environ["MAGE_TEST_ROOT"]
    pbs_options["mage_test_set_root"] = os.environ["MAGE_TEST_SET_ROOT"]
+    pbs_options["conda_environment"] = os.environ["CONDA_ENVIRONMENT"]
    pbs_options["kaijuhome"] = KAIJUHOME
-    pbs_options["kaipy_private_root"] = os.environ["KAIPY_PRIVATE_ROOT"]
    pbs_options["tmpdir"] = os.environ["TMPDIR"]
    pbs_options["slack_bot_token"] = os.environ["SLACK_BOT_TOKEN"]
    pbs_options["branch_or_commit"] = os.environ["BRANCH_OR_COMMIT"]
@@ -255,7 +279,6 @@ def mage_reproducibility_check(args: dict):
    pbs_options["make_cmd"] = make_cmd
    pbs_options["genLFM_cmd"] = genLFM_cmd
    pbs_options["cda2wind_cmd"] = cda2wind_cmd
-    pbs_options["genRCM_cmd"] = genRCM_cmd
    pbs_options["genRaiju_cmd"] = genRaiju_cmd
    pbs_options["mpiexec_cmd"] = mpiexec_cmd
    pbs_options["voltron_cmd"] = voltron_cmd
@@ -326,13 +349,10 @@ def mage_reproducibility_check(args: dict):
    pbs_options["job_priority"] = os.environ["DERECHO_TESTING_PRIORITY"]
    pbs_options["walltime"] = "02:00:00"
    pbs_options["modules"] = module_names
-    pbs_options["condarc"] = os.environ["CONDARC"]
-    pbs_options["conda_envs_path"] = os.environ["CONDA_ENVS_PATH"]
-    pbs_options["conda_environment"] = os.environ["CONDA_ENVIRONMENT"]
    pbs_options["mage_test_root"] = os.environ["MAGE_TEST_ROOT"]
    pbs_options["mage_test_set_root"] = os.environ["MAGE_TEST_SET_ROOT"]
+    pbs_options["conda_environment"] = os.environ["CONDA_ENVIRONMENT"]
    pbs_options["kaijuhome"] = KAIJUHOME
-    pbs_options["kaipy_private_root"] = os.environ["KAIPY_PRIVATE_ROOT"]
    pbs_options["tmpdir"] = os.environ["TMPDIR"]
    pbs_options["slack_bot_token"] = os.environ["SLACK_BOT_TOKEN"]
    pbs_options["branch_or_commit"] = os.environ["BRANCH_OR_COMMIT"]
@@ -403,7 +423,7 @@ def mage_reproducibility_check(args: dict):
 def main():
    """Driver for command-line version of code."""
    # Set up the command-line parser.
-    parser = common.create_command_line_parser(DESCRIPTION)
+    parser = create_command_line_parser()

 #     # Add additional arguments specific to this script.
 #     parser.add_argument(
--- a/testingScripts/relCasePost-template.pbs
+++ b/testingScripts/relCasePost-template.pbs
@@ -12,9 +12,9 @@
 echo "Job $PBS_JOBID started at `date` on `hostname` in directory `pwd`."

 echo 'Loading python environment.'
+mage_test_root=$HOME
 if [ -d "${mage_test_root}/miniconda3" ]; then
    echo 'Loading local miniconda3'
-    mage_test_root='{{ mage_test_root }}'
    export CONDARC="${mage_test_root}/.condarc"
    export CONDA_ENVS_PATH="${mage_test_root}/.conda"
    mage_miniconda3="${mage_test_root}/miniconda3"
@@ -54,7 +54,7 @@ echo 'The active environment variables are:'
 printenv

 # Process the data and generate the output video
-python $KAIPYHOME/kaipy/scripts/quicklook/gamerrVid.py -d1 {{ case1F }} -id1 {{ case1id }} -d2 {{ case2F }} -id2 {{ case2id }} -o {{ frameFolder }}/{{ caseName }} -ts {{ ts }} -te {{ te }} -dt {{ dt }} -Nth 9 >& {{ caseName }}.out
+gamerrVid -d1 {{ case1F }} -id1 {{ case1id }} -d2 {{ case2F }} -id2 {{ case2id }} -o {{ frameFolder }}/{{ caseName }} -ts {{ ts }} -te {{ te }} -dt {{ dt }} -Nth 9 >& {{ caseName }}.out

 # copy output video to test root folder
 cp {{ frameFolder }}/{{ caseName }}.mp4 $MAGE_TEST_SET_ROOT/.
--- a/testingScripts/relCaseReport-template.pbs
+++ b/testingScripts/relCaseReport-template.pbs
@@ -12,6 +12,7 @@
 echo "Job $PBS_JOBID started at `date` on `hostname` in directory `pwd`."

 echo 'Loading python environment.'
+mage_test_root=$HOME
 if [ -d "${mage_test_root}/miniconda3" ]; then
    echo 'Loading local miniconda3'
    mage_test_root='{{ mage_test_root }}'
--- a/testingScripts/relativeCase-template.pbs
+++ b/testingScripts/relativeCase-template.pbs
@@ -23,6 +23,7 @@ module load {{ module }}
 module list

 echo 'Loading python environment.'
+mage_test_root=$HOME
 if [ -d "${mage_test_root}/miniconda3" ]; then
    echo 'Loading local miniconda3'
    mage_test_root='{{ mage_test_root }}'
--- a/testingScripts/relativeCaseGo-template.xml
+++ b/testingScripts/relativeCaseGo-template.xml
@@ -3,7 +3,7 @@
 <Kaiju>
  <VOLTRON>
    <time tFin="7201.0"/>
-    <spinup doSpin="T" tSpin="3600.0"/>
+    <spinup doSpin="T" tSpin="3600.0" tIO="-600.0"/>
    <output dtOut="60.0" tsOut="100"/>
    <coupling dtCouple="5.0" imType="RAIJU" doQkSquish="F" qkSquishStride="2" doAsyncCoupling="F" doSerial="{{ serial_coupling }}"/>
    <restart dtRes="3600.0"/>
@@ -41,7 +41,7 @@
    <output loudConsole="T" doFat="F" doLossExtras="F" doDebug="F" writeGhosts="F"/>
    <grid gType="SHGRID" ThetaL="15" ThetaU="50"/>
    <domain tail_buffer="15.0" sun_buffer="15.0" tail_active="12.0" sun_active="12.0"/>
-    <sim pdmb="0.75"/>
+    <sim pdmb="0.75" doSmoothgrads="F"/>
    <config fname="raijuconfig.h5"/>
    <plasmasphere doPsphere="T" doExcessMap="T"/>
    <losses doLosses="T" doCX="T" doSS="T" doCC="T"/>
--- a/testingScripts/relativeTests.py
+++ b/testingScripts/relativeTests.py
@@ -39,9 +39,6 @@ TEST_DIRECTORY = os.path.join(MAGE_TEST_SET_ROOT, 'compTest')
 # Home directory of kaiju installation
 KAIJUHOME = os.environ['KAIJUHOME']

-# Home directory of kaipy installation
-KAIPYHOME = os.environ['KAIPYHOME']
-
 # Path to directory containing the test scripts
 TEST_SCRIPTS_DIRECTORY = os.path.join(KAIJUHOME, 'testingScripts')

@@ -610,7 +607,7 @@ def main():
        # Generate the LFM grid file.
        if verbose:
            print('Creating LFM grid file.')
-        cmd = 'genLFM.py -gid D'
+        cmd = 'genLFM -gid D'
        if debug:
            print(f"cmd = {cmd}")
        try:
@@ -620,7 +617,7 @@ def main():
                  f"{module_set_name}.\n"
                  f"e.cmd = {e.cmd}\n"
                  f"e.returncode = {e.returncode}\n"
-                  'See testing log for output from genLFM.py.\n'
+                  'See testing log for output from genLFM.\n'
                  'Skipping remaining steps for module set'
                  f"{module_set_name}\n")
            continue
@@ -628,7 +625,7 @@ def main():
        # Generate the solar wind boundary condition file.
        if verbose:
            print('Creating solar wind initial conditions file.')
-        cmd = 'cda2wind.py -t0 2016-08-09T02:00:00 -t1 2016-08-09T12:00:00'
+        cmd = 'cda2wind -t0 2016-08-09T02:00:00 -t1 2016-08-09T12:00:00'
        if debug:
            print(f"cmd = {cmd}")
        try:
@@ -638,7 +635,7 @@ def main():
                  f" for module set {module_set_name}.\n"
                  f"e.cmd = {e.cmd}\n"
                  f"e.returncode = {e.returncode}\n"
-                  'See testing log for output from cda2wind.py.\n'
+                  'See testing log for output from cda2wind.\n'
                  'Skipping remaining steps for module set'
                  f"{module_set_name}\n")
            continue
@@ -646,7 +643,7 @@ def main():
        # Generate the Raiju configuration file.
        if verbose:
            print('Creating Raiju configuration file.')
-        cmd = 'genRAIJU.py'
+        cmd = 'genRAIJU'
        if debug:
            print(f"cmd = {cmd}")
        try:
@@ -656,7 +653,7 @@ def main():
                  f" for module set {module_set_name}.\n"
                  f"e.cmd = {e.cmd}\n"
                  f"e.returncode = {e.returncode}\n"
-                  'See testing log for output from genRCM.py.\n'
+                  'See testing log for output from genRAIJU.\n'
                  'Skipping remaining steps for module set '
                  f"{module_set_name}\n")
            continue
@@ -668,7 +665,6 @@ def main():
        base_pbs_options['job_priority'] = os.environ['DERECHO_TESTING_PRIORITY']
        base_pbs_options['modules'] = module_names
        base_pbs_options['kaijuhome'] = KAIJUHOME
-        base_pbs_options['kaipyhome'] = KAIPYHOME
        base_pbs_options['tmpdir'] = os.environ['TMPDIR']
        base_pbs_options['slack_bot_token'] = os.environ['SLACK_BOT_TOKEN']
        base_pbs_options['mage_test_root'] = os.environ['MAGE_TEST_ROOT']
--- a/testingScripts/run_mage-template.pbs
+++ b/testingScripts/run_mage-template.pbs
@@ -19,11 +19,11 @@ module load {{ module }}
 echo 'The currently loaded modules are:'
 module list

+echo 'Loading python environment.'
+mage_test_root=$HOME
 if [ -d "${mage_test_root}/miniconda3" ]; then
    echo 'Loading local miniconda3'
-    export CONDARC="{{ condarc }}"
-    export CONDA_ENVS_PATH="{{ conda_envs_path }}"
-    mage_miniconda3="{{ mage_test_root }}/miniconda3"
+    mage_miniconda3="${mage_test_root}/miniconda3"
    mage_conda="${mage_miniconda3}/bin/conda"
    __conda_setup="$($mage_conda 'shell.bash' 'hook' 2> /dev/null)"
    if [ $? -eq 0 ]; then
@@ -47,10 +47,6 @@ echo 'Setting up MAGE environment.'
 source {{ kaijuhome }}/scripts/setupEnvironment.sh
 echo "The kaiju software is located at ${KAIJUHOME}."

-echo 'Setting up kaipy environment.'
-source {{ kaipy_private_root }}/kaipy/scripts/setupEnvironment.sh
-echo "The kaipy software is located at ${KAIPYHOME}."
-
 echo 'Setting environment variables.'
 export TMPDIR={{ tmpdir }}
 export SLACK_BOT_TOKEN={{ slack_bot_token }}
--- a/testingScripts/run_mage_tests.sh
+++ b/testingScripts/run_mage_tests.sh
@@ -0,0 +1,381 @@
+#!/usr/bin/bash
+
+# ############################################################################
+
+# IMPORTANT NOTES:
+
+# This bash script was designed to run from a cron job on the 'cron' host at
+# NCAR, using ssh to execute this script on a derecho login node. See
+# kaiju/testingScripts/crontab for an example of how to invoke this script.
+
+# SSH must be configured so that ssh from cron to derecho does not require a
+# password.
+
+# ############################################################################
+
+echo '***********************************************************************'
+echo "Starting $0 at `date` on `hostname`."
+
+# ############################################################################
+
+# Force this script to exit on any failure.
+set -e
+
+# ----------------------------------------------------------------------------
+
+# These *should* be the only variables you have to change when moving the
+# testing environment around or changing python environments.
+
+# Root of kaiju testing environment - *everything* goes under here, except
+# for the outputs from the individual test runs.
+export MAGE_TEST_ROOT='/glade/campaign/hao/msphere/automated_kaiju_tests'
+
+# Root of kaiju test results tree.
+export MAGE_TEST_RESULTS_ROOT='/glade/derecho/scratch/ewinter/mage_testing'
+
+# Location of the miniconda installation used for testing.
+export MAGE_MINICONDA='/glade/u/home/ewinter/miniconda3'
+
+# Setup command for conda.
+__conda_setup="$('/glade/u/home/ewinter/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)"
+
+# conda environment for testing
+export CONDA_ENVIRONMENT='kaiju-3.12-testing'
+
+# ----------------------------------------------------------------------------
+
+# Other exported environment variables
+
+# Path to directory containing the dateime-stamped directories for individual
+# sets of test runs.
+export MAGE_TEST_RUNS_ROOT="${MAGE_TEST_RESULTS_ROOT}/test_runs"
+
+# PBS account to use for running tests on derecho
+export DERECHO_TESTING_ACCOUNT='P28100045'
+
+# PBS queue to use for running tests on derecho
+export DERECHO_TESTING_QUEUE='main'
+
+# PBS priority to use for running tests on derecho
+export DERECHO_TESTING_PRIORITY='economy'
+
+# Set the token for sending messages to Slack.
+export SLACK_BOT_TOKEN=`cat $HOME/.ssh/slack.txt`
+
+# IMPORTANT: Set this environment variable to force the python print()
+# function to automatically flush its output in all of the testing scripts.
+# This will ensure that output from the testing scripts is logged in the order
+# that it is created.
+export PYTHONUNBUFFERED='TRUE'
+
+# ----------------------------------------------------------------------------
+
+# Non-exported variables used by this script
+
+# Path to the SSH key to use for running the tests. The key must have no
+# passphrase. This is needed to allow passwordless access to BitBucket.
+ssh_key_for_testing='/glade/u/home/ewinter/.ssh/id_rsa_kaiju_testing'
+
+# Setup script for CDF code
+cdf_setup_script="${MAGE_TEST_ROOT}/local/cdf/3.9.0/bin/definitions.B"
+
+# Address of kaiju-private repository on BitBucket.
+kaiju_repository='git@bitbucket.org:aplkaiju/kaiju-private.git'
+
+# Name of local directory containing clone of repository.
+local_kaiju_name='kaiju'
+
+# Default kaiju code branch to test if no branch, commit, or tag is specified.
+default_branch_to_test='development'
+
+# Output string to separate results from different tests.
+test_output_separator='------------------------------------------------------'
+
+# ############################################################################
+
+# Define the command-line help function.
+Help()
+{
+    # Display Help
+    echo 'Control script for running MAGE tests via cron and ssh.'
+    echo
+    echo "Syntax: run_mage_tests.sh [-b branch] [-c commit] [-d] [-h] [-v] 'test1[,test2,...]'"
+    echo 'options:'
+    echo 'b branch   Run tests using this branch'
+    echo 'c commit   Run tests using this commit (or tag)'
+    echo 'd          Use debug mode.'
+    echo 'h          Print this help message.'
+    echo 'v          Use verbose mode.'
+    echo
+    echo 'If no branch or commit is specified, the latest commit on the development branch will be used for the tests.'
+    echo "Each test can have its own options, e.g. 'buildTest.py -d,unitTest.py -lv'"
+}
+
+# Process command-line options.
+branch=''
+commit=''
+debug=false
+verbose=false
+while getopts ':b:c:dhv' option; do
+    case $option in
+        b) # Test a specific branch
+            branch=$OPTARG;;
+        c) # Test a specific commit or tag
+            commit=$OPTARG;;
+        d) # debug mode
+            debug=true;;
+        h) # display Help
+            Help
+            exit;;
+        v) # verbose mode
+            verbose=true;;
+        \?) # Invalid option
+            echo 'Error: Invalid option'
+            exit 1;;
+    esac
+done
+
+if $debug; then
+    echo "branch=${branch}"
+    echo "commit=${commit}"
+    echo "debug=${debug}"
+    echo "help=${help}"
+    echo "verbose=${verbose}"
+fi
+
+# Fetch the branch or commit to test. If neither specified, use the default.
+if [[ -n $branch && -n $commit ]]; then
+    echo 'Cannot specify branch and commit together!'
+    exit 1
+fi
+if [[ -z $branch && -z $commit ]]; then
+    branch=$default_branch_to_test
+fi
+
+# At this point, either branch is specified, or commit/tag is specified.
+# There should be no case where both are unspecified, or both are specified.
+if [[ -n $branch ]]; then
+    export BRANCH_OR_COMMIT=$branch
+else
+    export BRANCH_OR_COMMIT=$commit
+fi
+if $debug; then
+    echo "BRANCH_OR_COMMIT=${BRANCH_OR_COMMIT}"
+fi
+
+# Fetch the list of tests to run.
+tests_to_run_str=$BASH_ARGV
+if $debug; then
+    echo "tests_to_run_str=${tests_to_run_str}"
+fi
+
+# Split the test list string into an array.
+# Tests are separated by comma, no spaces.
+IFS="," tests_to_run=($tests_to_run_str)
+if $debug; then
+    echo 'The test commands to run are:'
+    for test_cmd in "${tests_to_run[@]}"
+    do
+        echo $test_cmd
+    done
+fi
+
+# ############################################################################
+
+# Load the SSH private key for testing. This is needed for BitBucket access.
+# Note that this key does not have a passphrase, so it can easily be used
+# from a cron job.
+if $verbose; then
+    echo 'Loading SSH key into key agent.'
+fi
+eval `ssh-agent -s`
+ssh-add $ssh_key_for_testing
+
+# ############################################################################
+
+# Set up the module system (needed when running from cron, harmless
+# otherwise).
+if $verbose; then
+    echo 'Setting up module system.'
+fi
+source /etc/profile.d/z00_modules.sh
+
+# List at-start modules.
+# The 2>&1 (no spaces!) is needed since the 'module' command sends
+# output to stderr by default, and so it is lost when stdout is sent
+# back to the user.
+if $verbose; then
+    echo 'At start, the loaded modules are:'
+    module list 2>&1
+fi
+
+# ############################################################################
+
+# Activate the conda installation for MAGE testing.
+if $verbose; then
+    echo "Setting up conda environment for MAGE testing ${CONDA_ENVIRONMENT}."
+fi
+
+# This code is based on the code that the miniconda installer puts into
+# ~/.bashrc or ~/.bash_profile when a user installs miniconda.
+# This code is needed since $HOME/.bashrc is not run.
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "${MAGE_MINICONDA}/etc/profile.d/conda.sh" ]; then
+        . "${MAGE_MINICONDA}/etc/profile.d/conda.sh"
+    else
+        export PATH="${MAGE_MINICONDA}/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+conda activate $CONDA_ENVIRONMENT
+if $verbose; then
+    echo "conda environment is `echo $CONDA_PREFIX`"
+fi
+
+if $verbose; then
+    echo 'The installed version of kaipy in this environment is:'
+    conda list kaipy
+fi
+
+# ############################################################################
+
+# Make the CDF library available (needed by the satellite comparison
+# python scripts).
+if $verbose; then
+    echo 'Sourcing CDF setup script.'
+fi
+source $cdf_setup_script
+
+# ############################################################################
+
+# Create the ISO 8601 datetime stamp for this set of tests.
+testing_datetime=`date --iso-8601=seconds`
+if $debug; then
+    echo "testing_datetime=${testing_datetime}"
+fi
+
+# <HACK>
+# The make tool cannot handle file or directory names that contain a colon
+# (':'). Process the ISO 8601 date time string from the ISO 8601 form:
+# YYYY-MM-DDTHH:mm:ss[+-]hh:mm
+# into the more compact form:
+# YYYYMMDD_HHMMSS
+# The sed commands to do this are, in order:
+# s/.\{6\}$// - Remove the time zone offset (last 6 characters)
+# s/-//g - Delete all '-'.
+# s/\://g - Delete all ':'.
+# s/T/_/ - Convert 'T' separating date and time to underscore.
+testing_datetime=`echo $testing_datetime | sed -e 's/.\{6\}$//' -e 's/-//g' -e 's/\://g' -e 's/T/_/'`
+if $debug; then
+    echo "testing_datetime=${testing_datetime}"
+fi
+# </HACK>
+
+# Create a directory to hold all of the tests for this set of tests.
+test_set_dir="${testing_datetime}-${BRANCH_OR_COMMIT}"
+if $debug; then
+    echo "test_set_dir=${test_set_dir}"
+fi
+export MAGE_TEST_SET_ROOT="${MAGE_TEST_RUNS_ROOT}/${test_set_dir}"
+if $verbose; then
+    echo "Creating directory for this set of tests at ${MAGE_TEST_SET_ROOT}."
+fi
+mkdir $MAGE_TEST_SET_ROOT
+
+# ############################################################################
+
+# Move to the directory for this set of tests.
+cd $MAGE_TEST_SET_ROOT
+
+# Clone the kaiju-private repository, call it kaiju locally.
+if $verbose; then
+    echo "Cloning repository ${kaiju_repository}."
+fi
+git clone $kaiju_repository $local_kaiju_name
+
+# Move into the repository clone.
+cd $local_kaiju_name
+
+# If a branch was requested, switch to that branch. Otherwise, if a commit
+# or tag was requested, check out that commit or tag.
+if $verbose; then
+    echo "Checking out branch/commit/tag ${BRANCH_OR_COMMIT}."
+fi
+git checkout $BRANCH_OR_COMMIT
+
+# ############################################################################
+
+# <HACK>
+# Back up existing test code, since we need to use test code that is under
+# development.
+mkdir TEST_CODE_BACKUP
+mv testingScripts TEST_CODE_BACKUP/
+mv tests TEST_CODE_BACKUP/
+
+# Copy latest test code.
+new_test_code_root="${MAGE_TEST_ROOT}/kaiju-private"
+if $verbose; then
+    echo "Copying updated test files from ${new_test_code_root}."
+fi
+cp -rp $new_test_code_root/testingScripts ./testingScripts
+cp -rp $new_test_code_root/tests ./tests
+# </HACK>
+
+# ############################################################################
+
+# Set up the kaiju environment using the just-checked-out code.
+kaiju_setup_script="${MAGE_TEST_SET_ROOT}/${local_kaiju_name}/scripts/setupEnvironment.sh"
+if $verbose; then
+    echo "Sourcing kaiju setup script ${kaiju_setup_script}."
+fi
+source $kaiju_setup_script
+if $verbose; then
+    echo "KAIJUHOME is ${KAIJUHOME}."
+fi
+
+# ############################################################################
+
+# List at-start environment variables.
+if $debug; then
+    echo 'At start, the environment variables are:'
+    printenv
+fi
+
+# ############################################################################
+
+# Compute the location of the test scripts.
+kaiju_test_scripts_dir="${KAIJUHOME}/testingScripts"
+if $debug; then
+    echo "kaiju_test_scripts_dir=${kaiju_test_scripts_dir}"
+fi
+
+# Run each test.
+for mage_test in ${tests_to_run[@]}
+do
+    echo $test_output_separator
+    if $verbose; then
+        echo "Moving to ${MAGE_TEST_SET_ROOT}."
+    fi
+    cd $MAGE_TEST_SET_ROOT
+    cmd="python ${kaiju_test_scripts_dir}/${mage_test}"
+    if $verbose; then
+        echo "Running test '${cmd}' at `date` on `hostname`."
+    fi
+    eval "${cmd}"
+done
+echo $test_output_separator
+
+# ############################################################################
+
+# Shut down the SSH agent.
+if $verbose; then
+    echo "Shutting down SSH key agent."
+fi
+ssh-agent -k
+
+# ############################################################################
+
+echo "Ending $0 at `date` on `hostname`."
+echo '***********************************************************************'
--- a/testingScripts/unitTest.py
+++ b/testingScripts/unitTest.py
@@ -145,7 +145,7 @@ UNIT_TEST_DATA_INPUT_FILES = [
    'bcwind.h5',
    'geo_mpi.xml',
    'lfmD.h5',
-    'rcmconfig.h5',
+    'raijuconfig.h5',
 ]

 # Names of PBS scripts to create from templates.
@@ -425,17 +425,17 @@ def main():
                    print(f"Copying {from_path} to {to_path}.")
                shutil.copyfile(from_path, to_path)
        else:
-            cmd = "cda2wind.py -t0 2016-08-09T09:00:00 -t1 2016-08-09T11:00:00"
+            cmd = "cda2wind -t0 2016-08-09T09:00:00 -t1 2016-08-09T11:00:00"
            if debug:
                print(f"cmd = {cmd}")
            cproc = subprocess.run(cmd, shell=True, check=True,
                                   text=True, capture_output=True)
-            cmd = "genLFM.py"
+            cmd = "genLFM"
            if debug:
                print(f"cmd = {cmd}")
            cproc = subprocess.run(cmd, shell=True, check=True,
                                   text=True, capture_output=True)
-            cmd = "genRAIJU.py"
+            cmd = "genRAIJU"
            if debug:
                print(f"cmd = {cmd}")
            cproc = subprocess.run(cmd, shell=True, check=True,
--- a/testingScripts/weeklyDash-template.pbs
+++ b/testingScripts/weeklyDash-template.pbs
@@ -20,9 +20,7 @@ echo 'The currently loaded modules are:'
 module list

 echo 'Loading python environment.'
-export CONDARC="{{ condarc }}"
-export CONDA_ENVS_PATH="{{ conda_envs_path }}"
-mage_miniconda3="{{ mage_test_root }}/miniconda3"
+mage_miniconda3="${HOME}/miniconda3"
 mage_conda="${mage_miniconda3}/bin/conda"
 __conda_setup="$($mage_conda 'shell.bash' 'hook' 2> /dev/null)"
 if [ $? -eq 0 ]; then
@@ -42,10 +40,6 @@ echo 'Setting up MAGE environment.'
 source {{ kaijuhome }}/scripts/setupEnvironment.sh
 echo "The kaiju software is located at ${KAIJUHOME}."

-echo 'Setting up kaipy environment.'
-source {{ kaipy_private_root }}/kaipy/scripts/setupEnvironment.sh
-echo "The kaipy software is located at ${KAIPYHOME}."
-
 echo 'Setting environment variables.'
 export TMPDIR={{ tmpdir }}
 export SLACK_BOT_TOKEN={{ slack_bot_token }}
@@ -83,10 +77,10 @@ echo 'Generating the solar wind boundary condition file.'
 {{ cda2wind_cmd }}
 echo "The solar wind boundary condition file is `ls bcwind.h5`."

-# Generate the RCM configuration file.
-echo 'Generating the RCM configuration file.'
-{{ genRCM_cmd }}
-echo "The RCM configuration file is `ls rcmconfig.h5`."
+# Generate the raiju configuration file.
+echo 'Generating the raiju configuration file.'
+{{ genRAIJU_cmd }}
+echo "The RAIJU configuration file is `ls raijuconfig.h5`."

 # Run the model.
 MPICOMMAND="{{ mpiexec_cmd }}"
--- a/testingScripts/weeklyDash.py
+++ b/testingScripts/weeklyDash.py
@@ -142,15 +142,15 @@ def weekly_dash(args: dict):
    make_cmd = "make voltron_mpi.x"

    # Create the command to generate the LFM grid.
-    genLFM_cmd = "genLFM.py -gid Q"
+    genLFM_cmd = "genLFM -gid Q"

    # Create the command to generate the solar wind boundary condition file.
    cda2wind_cmd = (
-        "cda2wind.py -t0 2016-08-09T02:00:00 -t1 2016-08-09T12:00:00"
+        "cda2wind -t0 2016-08-09T02:00:00 -t1 2016-08-09T12:00:00"
    )

-    # Create the command to generate the RCM configuration.
-    genRCM_cmd = "genRCM.py"
+    # Create the command to generate the raiju configuration.
+    genRAIJU_cmd = "genRAIJU"

    # Create the command for launching an MPI program.
    mpiexec_cmd = f"mpiexec {KAIJUHOME}/scripts/preproc/pinCpuCores.sh"
@@ -178,13 +178,10 @@ def weekly_dash(args: dict):
    pbs_options["job_priority"] = os.environ["DERECHO_TESTING_PRIORITY"]
    pbs_options["walltime"] = "08:00:00"
    pbs_options["modules"] = module_names
-    pbs_options["condarc"] = os.environ["CONDARC"]
-    pbs_options["conda_envs_path"] = os.environ["CONDA_ENVS_PATH"]
    pbs_options["conda_environment"] = os.environ["CONDA_ENVIRONMENT"]
    pbs_options["mage_test_root"] = os.environ["MAGE_TEST_ROOT"]
    pbs_options["mage_test_set_root"] = os.environ["MAGE_TEST_SET_ROOT"]
    pbs_options["kaijuhome"] = KAIJUHOME
-    pbs_options["kaipy_private_root"] = os.environ["KAIPY_PRIVATE_ROOT"]
    pbs_options["tmpdir"] = os.environ["TMPDIR"]
    pbs_options["slack_bot_token"] = os.environ["SLACK_BOT_TOKEN"]
    pbs_options["branch_or_commit"] = os.environ["BRANCH_OR_COMMIT"]
@@ -192,7 +189,7 @@ def weekly_dash(args: dict):
    pbs_options["make_cmd"] = make_cmd
    pbs_options["genLFM_cmd"] = genLFM_cmd
    pbs_options["cda2wind_cmd"] = cda2wind_cmd
-    pbs_options["genRCM_cmd"] = genRCM_cmd
+    pbs_options["genRAIJU_cmd"] = genRAIJU_cmd
    pbs_options["mpiexec_cmd"] = mpiexec_cmd
    pbs_options["voltron_cmd"] = voltron_cmd

--- a/testingScripts/weeklyDashReport.py
+++ b/testingScripts/weeklyDashReport.py
@@ -121,9 +121,6 @@ REMIX_NORTH_QUICKLOOK_MASTER = os.path.join(
 REMIX_SOUTH_QUICKLOOK_MASTER = os.path.join(
    REFERENCE_RESULTS_DIRECTORY_MASTER, 'remix_s.png'
 )
-RCM_QUICKLOOK_MASTER = os.path.join(
-    REFERENCE_RESULTS_DIRECTORY_MASTER, 'qkrcmpic.png'
-)

 # Compute the paths to the quicklook plots for the development branch.
 MAGNETOSPHERE_QUICKLOOK_DEVELOPMENT = os.path.join(
@@ -135,9 +132,6 @@ REMIX_NORTH_QUICKLOOK_DEVELOPMENT = os.path.join(
 REMIX_SOUTH_QUICKLOOK_DEVELOPMENT = os.path.join(
    REFERENCE_RESULTS_DIRECTORY_DEVELOPMENT, 'remix_s.png'
 )
-RCM_QUICKLOOK_DEVELOPMENT = os.path.join(
-    REFERENCE_RESULTS_DIRECTORY_DEVELOPMENT, 'qkrcmpic.png'
-)


 def main():
@@ -908,27 +902,6 @@ def main():

    # ------------------------------------------------------------------------

-    # Make the RCM quick-look plot.
-    if verbose:
-        print(f"Creating RCM quicklook plot for {os.getcwd()}.")
-
-    # Create the plot.
-    cmd = 'rcmpic.py'
-    if debug:
-        print(f"cmd = {cmd}")
-    try:
-        _ = subprocess.run(cmd, shell=True, check=True)
-    except subprocess.CalledProcessError as e:
-        print(
-            'ERROR: Unable to create RCM quicklook plot.\n'
-            f"e.cmd = {e.cmd}\n"
-            f"e.returncode = {e.returncode}\n"
-            f'See log for output.\n',
-            file=sys.stderr
-        )
-
-    # ------------------------------------------------------------------------
-
    # Create merged images for the quicklook plots.

    # Merge magnetosphere quicklooks.
@@ -988,25 +961,6 @@ def main():
            file=sys.stderr
        )

-    # Merge RCM quicklooks.
-    cmd = (
-        f"convert {RCM_QUICKLOOK_MASTER}"
-        f" {RCM_QUICKLOOK_DEVELOPMENT}"
-        ' qkrcmpic.png -append combined_qkrcmpic.png'
-    )
-    if debug:
-        print(f"cmd = {cmd}")
-    try:
-        cproc = subprocess.run(cmd, shell=True, check=True)
-    except subprocess.CalledProcessError as e:
-        print(
-            'ERROR: Unable to combine RCM quicklook plots.\n'
-            f"e.cmd = {e.cmd}\n"
-            f"e.returncode = {e.returncode}\n"
-            f'See log for output.\n',
-            file=sys.stderr
-        )
-
    # ------------------------------------------------------------------------

    # List the files to post and their comments.
@@ -1017,11 +971,9 @@ def main():
        'qkmsphpic.png',
        'remix_n.png',
        'remix_s.png',
-        'qkrcmpic.png',
        'combined_msphpic.png',
        'combined_remix_n.png',
        'combined_remix_s.png',
-        'combined_qkrcmpic.png'
    ]
    comments_to_post = [
        'Real-Time Performance\n\n',
@@ -1030,11 +982,9 @@ def main():
        'Magnetosphere Quicklook Plots\n\n',
        'REMIX (north) Quicklook Plots\n\n',
        'REMIX (south) Quicklook Plots\n\n',
-        'RCM Quicklook Plots\n\n',
        'Magnetosphere Quicklook Comparison Plots\n\n',
        'REMIX (north) Quicklook Comparison Plots\n\n',
        'REMIX (south) Quicklook Comparison Plots\n\n',
-        'RCM Quicklook Comparison Plots\n\n'
    ]

    # If loud mode is on, post results to Slack.
--- a/tests/unitTestReport-template.pbs
+++ b/tests/unitTestReport-template.pbs
@@ -19,11 +19,9 @@ module load {{ module }}
 module list

 echo 'Loading python environment.'
+mage_test_root=$HOME
 if [ -d "${mage_test_root}/miniconda3" ]; then
    echo 'Loading local miniconda3'
-    mage_test_root='{{ mage_test_root }}'
-    export CONDARC="${mage_test_root}/.condarc"
-    export CONDA_ENVS_PATH="${mage_test_root}/.conda"
    mage_miniconda3="${mage_test_root}/miniconda3"
    mage_conda="${mage_miniconda3}/bin/conda"
    __conda_setup="$($mage_conda 'shell.bash' 'hook' 2> /dev/null)"