From 728823b4f1384f30737b21e187dde7b94094b793 Mon Sep 17 00:00:00 2001 From: AnonNick Date: Tue, 9 Sep 2025 10:33:30 -0700 Subject: [PATCH 1/7] pleiades to atiken --- .../makeitso-gamhelio/option_descriptions.json | 14 +++++++------- scripts/makeitso/engage.py | 8 ++++---- scripts/makeitso/makeitso.py | 2 +- scripts/makeitso/option_descriptions.json | 18 +++++++++--------- .../makeitso/option_engage_descriptions.json | 2 +- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/scripts/makeitso-gamhelio/option_descriptions.json b/scripts/makeitso-gamhelio/option_descriptions.json index 1bdbeb13..cf9c6385 100644 --- a/scripts/makeitso-gamhelio/option_descriptions.json +++ b/scripts/makeitso-gamhelio/option_descriptions.json @@ -35,8 +35,8 @@ "hpc_system": { "LEVEL": "BASIC", "prompt": "Name of HPC system", - "default": "pleiades", - "valids": ["derecho", "pleiades"] + "default": "atiken", + "valids": ["derecho", "atiken"] } }, @@ -129,7 +129,7 @@ "default": "mpiexec pinCpuCores.sh" } }, - "pleiades": { + "atiken": { "queue": { "LEVEL": "BASIC", "prompt": "PBS queue name", @@ -149,7 +149,7 @@ "ncpus": { "LEVEL": "EXPERT", "prompt": "Number of cores per node", - "default": "28" + "default": "128" }, "mpiprocs": { "LEVEL": "EXPERT", @@ -159,12 +159,12 @@ "ompthreads": { "LEVEL": "EXPERT", "prompt": "Number of OMP threads per MPI rank", - "default": "14" + "default": "126" }, "other": { "LEVEL": "EXPERT", "prompt": "Additional options for PBS -l", - "default": ":model=bro" + "default": ":model=mil_ait" }, "modules": { "LEVEL": "EXPERT", @@ -460,7 +460,7 @@ "prompt": "(GAMERA) Block halo MPI", "default": { "derecho": "T", - "pleiades": "F" + "atiken": "F" }, "valids": ["T", "F"] } diff --git a/scripts/makeitso/engage.py b/scripts/makeitso/engage.py index 1d8218d3..c1ce9dc3 100755 --- a/scripts/makeitso/engage.py +++ b/scripts/makeitso/engage.py @@ -163,7 +163,7 @@ def create_pbs_scripts(gr_options: dict, makeitso_options:dict,makeitso_pbs_scri options["pbs"]["tie_mpiprocs"] = tiegcm_options["job"]["resource"]["mpiprocs"] options["pbs"]["tie_mpiranks"] = tiegcm_options["job"]["nprocs"] options["pbs"]["tie_exe"] = tiegcm_options["model"]["data"]["coupled_modelexe"] - if tiegcm_options["simulation"]["hpc_system"] == "pleiades": + if tiegcm_options["simulation"]["hpc_system"] == "atiken": options["pbs"]["model"] = tiegcm_options["job"]["resource"]["model"] # GR PBS parameters @@ -181,7 +181,7 @@ def create_pbs_scripts(gr_options: dict, makeitso_options:dict,makeitso_pbs_scri if tiegcm_options["simulation"]["hpc_system"] == "derecho": options["pbs"]["mpiexec_command"] = "mpiexec" options["pbs"]["mpiexec_option"] = "-n" - elif tiegcm_options["simulation"]["hpc_system"] == "pleiades": + elif tiegcm_options["simulation"]["hpc_system"] == "atiken": options["pbs"]["mpiexec_command"] = "mpiexec_mpt" options["pbs"]["mpiexec_option"] = "-np" options["pbs"]["tie_scripts"] = "correctOMPenvironment.sh $NODEFILE_1 omplace" @@ -402,7 +402,7 @@ def prompt_user_for_run_options(args): od["num_helpers"]["default"][gamera_grid_type] ) od["modules"] = oed["modules"] - if hpc_platform == "pleiades": + if hpc_platform == "atiken": od["moduledir"] = oed["moduledir"] od["local_modules"] = oed["local_modules"] for on in od: @@ -495,7 +495,7 @@ def main(): od["num_helpers"]["default"][gamera_grid_type] ) od["modules"] = oed["modules"] - if hpc_platform == "pleiades": + if hpc_platform == "atiken": od["moduledir"] = oed["moduledir"] od["local_modules"] = oed["local_modules"] for on in od: diff --git a/scripts/makeitso/makeitso.py b/scripts/makeitso/makeitso.py index 6075554d..3b2e701b 100644 --- a/scripts/makeitso/makeitso.py +++ b/scripts/makeitso/makeitso.py @@ -254,7 +254,7 @@ def update_option_descriptions(option_descriptions: dict, args: dict): od["default"] = pbs[k] # Incorporate HPC platform-specific PBS options from engage in default. option_descriptions["pbs"][hpc_platform]["modules"]["default"] = pbs["modules"] - if hpc_platform == "pleiades": + if hpc_platform == "atiken": option_descriptions["pbs"][hpc_platform]["moduledir"]["default"] = pbs["moduledir"] option_descriptions["pbs"][hpc_platform]["local_modules"]["default"] = pbs["local_modules"] # Return the option descriptions. diff --git a/scripts/makeitso/option_descriptions.json b/scripts/makeitso/option_descriptions.json index 9b31c0a3..6ea9b7c4 100644 --- a/scripts/makeitso/option_descriptions.json +++ b/scripts/makeitso/option_descriptions.json @@ -57,8 +57,8 @@ "hpc_system": { "LEVEL": "BASIC", "prompt": "Name of HPC system", - "default": "pleiades", - "valids": ["derecho", "pleiades"] + "default": "atiken", + "valids": ["derecho", "atiken"] } }, @@ -181,7 +181,7 @@ "default": "mpiexec pinCpuCores.sh" } }, - "pleiades": { + "atiken": { "group_list": { "LEVEL": "BASIC", "prompt": "PBS group list", @@ -211,7 +211,7 @@ "ncpus": { "LEVEL": "EXPERT", "prompt": "Number of cores per node", - "default": "28" + "default": "128" }, "mpiprocs": { "LEVEL": "EXPERT", @@ -221,7 +221,7 @@ "ompthreads": { "LEVEL": "EXPERT", "prompt": "Number of OMP threads per MPI rank", - "default": "14" + "default": "64" }, "num_helpers": { "LEVEL": "EXPERT", @@ -241,12 +241,12 @@ "helper_ompthreads": { "LEVEL": "EXPERT", "prompt": "Number of OMP threads per helper node", - "default": "28" + "default": "128" }, "other": { "LEVEL": "EXPERT", "prompt": "Additional options for PBS -l", - "default": ":model=bro" + "default": ":model=mil_ait" }, "modules": { "LEVEL": "EXPERT", @@ -488,7 +488,7 @@ "prompt": "(GAMERA) Block halo MPI", "default": { "derecho": "T", - "pleiades": "F" + "atiken": "F" }, "valids": ["T", "F"] } @@ -574,7 +574,7 @@ "prompt": "(VOLTRON) Perform asynchronous model coupling", "default": { "derecho": "F", - "pleiades": "T" + "atiken": "T" }, "valids": ["T", "F"] } diff --git a/scripts/makeitso/option_engage_descriptions.json b/scripts/makeitso/option_engage_descriptions.json index 70d549f7..7e901870 100644 --- a/scripts/makeitso/option_engage_descriptions.json +++ b/scripts/makeitso/option_engage_descriptions.json @@ -54,7 +54,7 @@ ] } }, - "pleiades": { + "atiken": { "modules": { "LEVEL": "EXPERT", "prompt": "Modules to load", From b6882630d5ce12be2feb0d2fa74777a92c7add77 Mon Sep 17 00:00:00 2001 From: AnonNick Date: Thu, 11 Sep 2025 06:42:07 -0700 Subject: [PATCH 2/7] atiken runtime env update --- scripts/makeitso/option_engage_descriptions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/makeitso/option_engage_descriptions.json b/scripts/makeitso/option_engage_descriptions.json index 7e901870..3ec8aadc 100644 --- a/scripts/makeitso/option_engage_descriptions.json +++ b/scripts/makeitso/option_engage_descriptions.json @@ -78,7 +78,7 @@ "LEVEL": "EXPERT", "prompt": "Commands to add local module PATH", "default": [ - "export PREFIX=/u/nrao3/local3", + "export PREFIX=${HOME}/local_atiken", "export LIBRARY_PATH=${LIBRARY_PATH}:$PREFIX/lib", "export LD_LIBRARY_PATH=$LIBRARY_PATH", "export CPATH=$PREFIX/include", From f1e556f8c0121060a4f6c1cac4297219e30fefbc Mon Sep 17 00:00:00 2001 From: AnonNick Date: Thu, 11 Sep 2025 08:06:23 -0700 Subject: [PATCH 3/7] Switch to nrao3 built lib --- scripts/makeitso/option_engage_descriptions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/makeitso/option_engage_descriptions.json b/scripts/makeitso/option_engage_descriptions.json index 3ec8aadc..8b9f98c6 100644 --- a/scripts/makeitso/option_engage_descriptions.json +++ b/scripts/makeitso/option_engage_descriptions.json @@ -78,7 +78,7 @@ "LEVEL": "EXPERT", "prompt": "Commands to add local module PATH", "default": [ - "export PREFIX=${HOME}/local_atiken", + "export PREFIX=/u/nrao3/local_atiken", "export LIBRARY_PATH=${LIBRARY_PATH}:$PREFIX/lib", "export LD_LIBRARY_PATH=$LIBRARY_PATH", "export CPATH=$PREFIX/include", From 5585b7e02b3164d4af0a414073dc5ed48eddefbc Mon Sep 17 00:00:00 2001 From: AnonNick Date: Mon, 13 Oct 2025 11:39:16 -0700 Subject: [PATCH 4/7] year boundary + aitken fix --- scripts/makeitso/engage.py | 12 ++++-- scripts/makeitso/makeitso.py | 43 +++++++++++++++++-- scripts/makeitso/option_descriptions.json | 10 ++--- .../makeitso/option_engage_descriptions.json | 4 +- 4 files changed, 55 insertions(+), 14 deletions(-) diff --git a/scripts/makeitso/engage.py b/scripts/makeitso/engage.py index c1ce9dc3..04c05c3a 100755 --- a/scripts/makeitso/engage.py +++ b/scripts/makeitso/engage.py @@ -163,7 +163,7 @@ def create_pbs_scripts(gr_options: dict, makeitso_options:dict,makeitso_pbs_scri options["pbs"]["tie_mpiprocs"] = tiegcm_options["job"]["resource"]["mpiprocs"] options["pbs"]["tie_mpiranks"] = tiegcm_options["job"]["nprocs"] options["pbs"]["tie_exe"] = tiegcm_options["model"]["data"]["coupled_modelexe"] - if tiegcm_options["simulation"]["hpc_system"] == "atiken": + if tiegcm_options["simulation"]["hpc_system"] == "aitken": options["pbs"]["model"] = tiegcm_options["job"]["resource"]["model"] # GR PBS parameters @@ -181,7 +181,7 @@ def create_pbs_scripts(gr_options: dict, makeitso_options:dict,makeitso_pbs_scri if tiegcm_options["simulation"]["hpc_system"] == "derecho": options["pbs"]["mpiexec_command"] = "mpiexec" options["pbs"]["mpiexec_option"] = "-n" - elif tiegcm_options["simulation"]["hpc_system"] == "atiken": + elif tiegcm_options["simulation"]["hpc_system"] == "aitken": options["pbs"]["mpiexec_command"] = "mpiexec_mpt" options["pbs"]["mpiexec_option"] = "-np" options["pbs"]["tie_scripts"] = "correctOMPenvironment.sh $NODEFILE_1 omplace" @@ -221,7 +221,9 @@ echo "Running tiegcm and voltron at the same time" # Create a PBS script for each segment. pbs_scripts = [] + print(f'Creating {options["pbs"]["num_segments"]} PBS scripts') for job in range(1,int(options["pbs"]["num_segments"])+1): + opt = copy.deepcopy(options) # Need a copy of options runid = opt["simulation"]["job_name"] segment_id = f"{runid}-{job:02d}" @@ -402,7 +404,7 @@ def prompt_user_for_run_options(args): od["num_helpers"]["default"][gamera_grid_type] ) od["modules"] = oed["modules"] - if hpc_platform == "atiken": + if hpc_platform == "aitken": od["moduledir"] = oed["moduledir"] od["local_modules"] = oed["local_modules"] for on in od: @@ -495,7 +497,7 @@ def main(): od["num_helpers"]["default"][gamera_grid_type] ) od["modules"] = oed["modules"] - if hpc_platform == "atiken": + if hpc_platform == "aitken": od["moduledir"] = oed["moduledir"] od["local_modules"] = oed["local_modules"] for on in od: @@ -627,6 +629,8 @@ def main(): with open('makeitso_parameters.json', 'w') as f: json.dump(makeitso_options, f, indent=JSON_INDENT) + # Update engage options with makeitso options + engage_options["pbs"]["num_segments"] = makeitso_options["pbs"]["num_segments"] # Run the TIEGCMrun coupled_options = copy.deepcopy(engage_options) diff --git a/scripts/makeitso/makeitso.py b/scripts/makeitso/makeitso.py index 3b2e701b..d0f36137 100644 --- a/scripts/makeitso/makeitso.py +++ b/scripts/makeitso/makeitso.py @@ -254,7 +254,7 @@ def update_option_descriptions(option_descriptions: dict, args: dict): od["default"] = pbs[k] # Incorporate HPC platform-specific PBS options from engage in default. option_descriptions["pbs"][hpc_platform]["modules"]["default"] = pbs["modules"] - if hpc_platform == "atiken": + if hpc_platform == "aitken": option_descriptions["pbs"][hpc_platform]["moduledir"]["default"] = pbs["moduledir"] option_descriptions["pbs"][hpc_platform]["local_modules"]["default"] = pbs["local_modules"] # Return the option descriptions. @@ -987,10 +987,27 @@ def create_ini_files(options: dict, args: dict): num_warmup_segments = i_last_warmup_ini # Create an .ini file for each simulation segment. Files for each # segment will be numbered starting with 1. + start_dt = datetime.datetime.fromisoformat(options["simulation"]["start_date"]) if "coupling" in args: num_segments = math.ceil((simulation_duration - num_warmup_segments*warmup_segment_duration)/segment_duration) + #start_dt = datetime.datetime.fromisoformat(options["simulation"]["start_date"]) + for job in range(1, num_segments + 1): + dT = float(options["simulation"]["segment_duration"]) + tfin_delta = float(coupling["gr_warm_up_time"]) + tStart_segment = (job - 1)*dT + tfin_delta + tFin_segment = job*dT + tfin_delta + 1.0 + # Check if the segment end time is exactly Jan 1, 00:00:00 of any year + segment_start_dt = start_dt + datetime.timedelta(seconds=tStart_segment) + segment_end_dt = start_dt + datetime.timedelta(seconds=tFin_segment-1) + if (segment_start_dt).year != (segment_end_dt).year: + if segment_end_dt.month != 1 or segment_end_dt.day != 1 or segment_end_dt.hour != 0 or segment_end_dt.minute != 0 or segment_end_dt.second != 0: + num_segments += 1 + break + else: num_segments = int(options["pbs"]["num_segments"]) + job_offset = 0 + tFin_offset = 0 for job in range(1, num_segments + 1): opt = copy.deepcopy(options) # Need a copy of options runid = opt["simulation"]["job_name"] @@ -1007,7 +1024,22 @@ def create_ini_files(options: dict, args: dict): else: tfin_delta = 0.0 # Add 1 to ensure last restart file created - tFin_segment = job*dT + tfin_delta + 1.0 + tStart_segment = (job - 1)*dT + tfin_delta + tFin_offset + + tFin_segment = (job - job_offset)*dT + tfin_delta + 1.0 + + segment_start_dt = start_dt + datetime.timedelta(seconds=tStart_segment) + segment_end_dt = start_dt + datetime.timedelta(seconds=tFin_segment-1) + if "coupling" in args and segment_start_dt.year != segment_end_dt.year: + if segment_end_dt.month != 1 or segment_end_dt.day != 1 or segment_end_dt.hour != 0 or segment_end_dt.minute != 0 or segment_end_dt.second != 0: + print(f'Segment {job} crosses year boundary, adjusting segment end time to Jan 1 of next year.') + next_year = segment_start_dt.year + 1 + jan1_next_year = datetime.datetime(next_year, 1, 1, 0, 0, 0) + tFin_segment = (jan1_next_year - start_dt).total_seconds() + 1.0 + tFin_offset = tFin_segment - (job - job_offset)*dT - tfin_delta - 1.0 + job_offset = 1 + segment_end_dt = start_dt + datetime.timedelta(seconds=tFin_segment - 1.0) + nRes = int(((tFin_segment - 1) - dT )/dtRes) opt["gamera"]["restart"]["nRes"] = str(nRes) # Engage modifications to parameters in coupled segment. @@ -1019,7 +1051,9 @@ def create_ini_files(options: dict, args: dict): else: # Subtract 1 from tFin padding for coupling beacuse to offset the +1.0 for restart file done above. tfin_padding = tfin_coupling_padding - 1.0 - opt["voltron"]["time"]["tFin"] = str(tFin_segment + tfin_padding) + + tFin = tFin_segment + tfin_padding + opt["voltron"]["time"]["tFin"] = str(tFin) #print(f'Creating job {job} with tFin_seg = {opt["voltron"]["time"]["tFin"]}') ini_content = template.render(opt) ini_file = os.path.join(opt["pbs"]["run_directory"], @@ -1380,6 +1414,9 @@ def makeitso(args: dict = None): f"script {all_jobs_script} like this:\n" f"bash {all_jobs_script}") + # Update number of segments for coupling in case of year rollover segments were added. + if "coupling" in args: + options["pbs"]["num_segments"] = str(len(pbs_scripts)-len(spinup_pbs_scripts)-len(warmup_pbs_scripts)) # Return the options dict used to create the PBS scripts, and the list # of PBS scripts which constitute the warmup period. return options, spinup_pbs_scripts, warmup_pbs_scripts diff --git a/scripts/makeitso/option_descriptions.json b/scripts/makeitso/option_descriptions.json index 6ea9b7c4..7d2a9dc4 100644 --- a/scripts/makeitso/option_descriptions.json +++ b/scripts/makeitso/option_descriptions.json @@ -57,8 +57,8 @@ "hpc_system": { "LEVEL": "BASIC", "prompt": "Name of HPC system", - "default": "atiken", - "valids": ["derecho", "atiken"] + "default": "aitken", + "valids": ["derecho", "aitken"] } }, @@ -181,7 +181,7 @@ "default": "mpiexec pinCpuCores.sh" } }, - "atiken": { + "aitken": { "group_list": { "LEVEL": "BASIC", "prompt": "PBS group list", @@ -488,7 +488,7 @@ "prompt": "(GAMERA) Block halo MPI", "default": { "derecho": "T", - "atiken": "F" + "aitken": "F" }, "valids": ["T", "F"] } @@ -574,7 +574,7 @@ "prompt": "(VOLTRON) Perform asynchronous model coupling", "default": { "derecho": "F", - "atiken": "T" + "aitken": "T" }, "valids": ["T", "F"] } diff --git a/scripts/makeitso/option_engage_descriptions.json b/scripts/makeitso/option_engage_descriptions.json index 8b9f98c6..49bf4faf 100644 --- a/scripts/makeitso/option_engage_descriptions.json +++ b/scripts/makeitso/option_engage_descriptions.json @@ -54,7 +54,7 @@ ] } }, - "atiken": { + "aitken": { "modules": { "LEVEL": "EXPERT", "prompt": "Modules to load", @@ -78,7 +78,7 @@ "LEVEL": "EXPERT", "prompt": "Commands to add local module PATH", "default": [ - "export PREFIX=/u/nrao3/local_atiken", + "export PREFIX=/u/nrao3/local_aitken", "export LIBRARY_PATH=${LIBRARY_PATH}:$PREFIX/lib", "export LD_LIBRARY_PATH=$LIBRARY_PATH", "export CPATH=$PREFIX/include", From f39d9d02ed9c7bc5b12ff58ce6ade33179037409 Mon Sep 17 00:00:00 2001 From: AnonNick Date: Mon, 13 Oct 2025 11:40:19 -0700 Subject: [PATCH 5/7] gamhelio aitken fix --- scripts/makeitso-gamhelio/option_descriptions.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/makeitso-gamhelio/option_descriptions.json b/scripts/makeitso-gamhelio/option_descriptions.json index 1d34ca1a..446d0851 100644 --- a/scripts/makeitso-gamhelio/option_descriptions.json +++ b/scripts/makeitso-gamhelio/option_descriptions.json @@ -35,8 +35,8 @@ "hpc_system": { "LEVEL": "BASIC", "prompt": "Name of HPC system", - "default": "atiken", - "valids": ["derecho", "atiken"] + "default": "aitken", + "valids": ["derecho", "aitken"] } }, @@ -129,7 +129,7 @@ "default": "mpiexec pinCpuCores.sh" } }, - "atiken": { + "aitken": { "queue": { "LEVEL": "BASIC", "prompt": "PBS queue name", @@ -455,7 +455,7 @@ "prompt": "(GAMERA) Block halo MPI", "default": { "derecho": "T", - "atiken": "F" + "aitken": "F" }, "valids": ["T", "F"] } From f09e972cf2cebcccfae450df0d4b1d86497790f2 Mon Sep 17 00:00:00 2001 From: AnonNick Date: Tue, 14 Oct 2025 06:22:47 -0700 Subject: [PATCH 6/7] gamhelio od.json typo fix --- scripts/makeitso-gamhelio/option_descriptions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/makeitso-gamhelio/option_descriptions.json b/scripts/makeitso-gamhelio/option_descriptions.json index 446d0851..7f982b15 100644 --- a/scripts/makeitso-gamhelio/option_descriptions.json +++ b/scripts/makeitso-gamhelio/option_descriptions.json @@ -159,7 +159,7 @@ "ompthreads": { "LEVEL": "EXPERT", "prompt": "Number of OMP threads per MPI rank", - "default": "126" + "default": "64" }, "other": { "LEVEL": "EXPERT", From a386f4d9b94d84d182b8461900dcac95cc16f9d6 Mon Sep 17 00:00:00 2001 From: AnonNick Date: Tue, 14 Oct 2025 06:28:10 -0700 Subject: [PATCH 7/7] Updated makeitso.py comments --- scripts/makeitso/makeitso.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/makeitso/makeitso.py b/scripts/makeitso/makeitso.py index d0f36137..aa5cd14a 100755 --- a/scripts/makeitso/makeitso.py +++ b/scripts/makeitso/makeitso.py @@ -990,13 +990,15 @@ def create_ini_files(options: dict, args: dict): start_dt = datetime.datetime.fromisoformat(options["simulation"]["start_date"]) if "coupling" in args: num_segments = math.ceil((simulation_duration - num_warmup_segments*warmup_segment_duration)/segment_duration) - #start_dt = datetime.datetime.fromisoformat(options["simulation"]["start_date"]) for job in range(1, num_segments + 1): dT = float(options["simulation"]["segment_duration"]) tfin_delta = float(coupling["gr_warm_up_time"]) tStart_segment = (job - 1)*dT + tfin_delta tFin_segment = job*dT + tfin_delta + 1.0 - # Check if the segment end time is exactly Jan 1, 00:00:00 of any year + # Check if the segment end time is exactly Jan 1, 00:00:00 of any year + # (Helps with year boundary issues in TIEGCM). + # If so, adjust the segment end time to be Jan 1 of the next year. + # This is only done if the segment start and end times are in different years. segment_start_dt = start_dt + datetime.timedelta(seconds=tStart_segment) segment_end_dt = start_dt + datetime.timedelta(seconds=tFin_segment-1) if (segment_start_dt).year != (segment_end_dt).year: