diff --git a/scripts/makeitso/engage.py b/scripts/makeitso/engage.py index c1ce9dc3..04c05c3a 100755 --- a/scripts/makeitso/engage.py +++ b/scripts/makeitso/engage.py @@ -163,7 +163,7 @@ def create_pbs_scripts(gr_options: dict, makeitso_options:dict,makeitso_pbs_scri options["pbs"]["tie_mpiprocs"] = tiegcm_options["job"]["resource"]["mpiprocs"] options["pbs"]["tie_mpiranks"] = tiegcm_options["job"]["nprocs"] options["pbs"]["tie_exe"] = tiegcm_options["model"]["data"]["coupled_modelexe"] - if tiegcm_options["simulation"]["hpc_system"] == "atiken": + if tiegcm_options["simulation"]["hpc_system"] == "aitken": options["pbs"]["model"] = tiegcm_options["job"]["resource"]["model"] # GR PBS parameters @@ -181,7 +181,7 @@ def create_pbs_scripts(gr_options: dict, makeitso_options:dict,makeitso_pbs_scri if tiegcm_options["simulation"]["hpc_system"] == "derecho": options["pbs"]["mpiexec_command"] = "mpiexec" options["pbs"]["mpiexec_option"] = "-n" - elif tiegcm_options["simulation"]["hpc_system"] == "atiken": + elif tiegcm_options["simulation"]["hpc_system"] == "aitken": options["pbs"]["mpiexec_command"] = "mpiexec_mpt" options["pbs"]["mpiexec_option"] = "-np" options["pbs"]["tie_scripts"] = "correctOMPenvironment.sh $NODEFILE_1 omplace" @@ -221,7 +221,9 @@ echo "Running tiegcm and voltron at the same time" # Create a PBS script for each segment. pbs_scripts = [] + print(f'Creating {options["pbs"]["num_segments"]} PBS scripts') for job in range(1,int(options["pbs"]["num_segments"])+1): + opt = copy.deepcopy(options) # Need a copy of options runid = opt["simulation"]["job_name"] segment_id = f"{runid}-{job:02d}" @@ -402,7 +404,7 @@ def prompt_user_for_run_options(args): od["num_helpers"]["default"][gamera_grid_type] ) od["modules"] = oed["modules"] - if hpc_platform == "atiken": + if hpc_platform == "aitken": od["moduledir"] = oed["moduledir"] od["local_modules"] = oed["local_modules"] for on in od: @@ -495,7 +497,7 @@ def main(): od["num_helpers"]["default"][gamera_grid_type] ) od["modules"] = oed["modules"] - if hpc_platform == "atiken": + if hpc_platform == "aitken": od["moduledir"] = oed["moduledir"] od["local_modules"] = oed["local_modules"] for on in od: @@ -627,6 +629,8 @@ def main(): with open('makeitso_parameters.json', 'w') as f: json.dump(makeitso_options, f, indent=JSON_INDENT) + # Update engage options with makeitso options + engage_options["pbs"]["num_segments"] = makeitso_options["pbs"]["num_segments"] # Run the TIEGCMrun coupled_options = copy.deepcopy(engage_options) diff --git a/scripts/makeitso/makeitso.py b/scripts/makeitso/makeitso.py index 3b2e701b..d0f36137 100644 --- a/scripts/makeitso/makeitso.py +++ b/scripts/makeitso/makeitso.py @@ -254,7 +254,7 @@ def update_option_descriptions(option_descriptions: dict, args: dict): od["default"] = pbs[k] # Incorporate HPC platform-specific PBS options from engage in default. option_descriptions["pbs"][hpc_platform]["modules"]["default"] = pbs["modules"] - if hpc_platform == "atiken": + if hpc_platform == "aitken": option_descriptions["pbs"][hpc_platform]["moduledir"]["default"] = pbs["moduledir"] option_descriptions["pbs"][hpc_platform]["local_modules"]["default"] = pbs["local_modules"] # Return the option descriptions. @@ -987,10 +987,27 @@ def create_ini_files(options: dict, args: dict): num_warmup_segments = i_last_warmup_ini # Create an .ini file for each simulation segment. Files for each # segment will be numbered starting with 1. + start_dt = datetime.datetime.fromisoformat(options["simulation"]["start_date"]) if "coupling" in args: num_segments = math.ceil((simulation_duration - num_warmup_segments*warmup_segment_duration)/segment_duration) + #start_dt = datetime.datetime.fromisoformat(options["simulation"]["start_date"]) + for job in range(1, num_segments + 1): + dT = float(options["simulation"]["segment_duration"]) + tfin_delta = float(coupling["gr_warm_up_time"]) + tStart_segment = (job - 1)*dT + tfin_delta + tFin_segment = job*dT + tfin_delta + 1.0 + # Check if the segment end time is exactly Jan 1, 00:00:00 of any year + segment_start_dt = start_dt + datetime.timedelta(seconds=tStart_segment) + segment_end_dt = start_dt + datetime.timedelta(seconds=tFin_segment-1) + if (segment_start_dt).year != (segment_end_dt).year: + if segment_end_dt.month != 1 or segment_end_dt.day != 1 or segment_end_dt.hour != 0 or segment_end_dt.minute != 0 or segment_end_dt.second != 0: + num_segments += 1 + break + else: num_segments = int(options["pbs"]["num_segments"]) + job_offset = 0 + tFin_offset = 0 for job in range(1, num_segments + 1): opt = copy.deepcopy(options) # Need a copy of options runid = opt["simulation"]["job_name"] @@ -1007,7 +1024,22 @@ def create_ini_files(options: dict, args: dict): else: tfin_delta = 0.0 # Add 1 to ensure last restart file created - tFin_segment = job*dT + tfin_delta + 1.0 + tStart_segment = (job - 1)*dT + tfin_delta + tFin_offset + + tFin_segment = (job - job_offset)*dT + tfin_delta + 1.0 + + segment_start_dt = start_dt + datetime.timedelta(seconds=tStart_segment) + segment_end_dt = start_dt + datetime.timedelta(seconds=tFin_segment-1) + if "coupling" in args and segment_start_dt.year != segment_end_dt.year: + if segment_end_dt.month != 1 or segment_end_dt.day != 1 or segment_end_dt.hour != 0 or segment_end_dt.minute != 0 or segment_end_dt.second != 0: + print(f'Segment {job} crosses year boundary, adjusting segment end time to Jan 1 of next year.') + next_year = segment_start_dt.year + 1 + jan1_next_year = datetime.datetime(next_year, 1, 1, 0, 0, 0) + tFin_segment = (jan1_next_year - start_dt).total_seconds() + 1.0 + tFin_offset = tFin_segment - (job - job_offset)*dT - tfin_delta - 1.0 + job_offset = 1 + segment_end_dt = start_dt + datetime.timedelta(seconds=tFin_segment - 1.0) + nRes = int(((tFin_segment - 1) - dT )/dtRes) opt["gamera"]["restart"]["nRes"] = str(nRes) # Engage modifications to parameters in coupled segment. @@ -1019,7 +1051,9 @@ def create_ini_files(options: dict, args: dict): else: # Subtract 1 from tFin padding for coupling beacuse to offset the +1.0 for restart file done above. tfin_padding = tfin_coupling_padding - 1.0 - opt["voltron"]["time"]["tFin"] = str(tFin_segment + tfin_padding) + + tFin = tFin_segment + tfin_padding + opt["voltron"]["time"]["tFin"] = str(tFin) #print(f'Creating job {job} with tFin_seg = {opt["voltron"]["time"]["tFin"]}') ini_content = template.render(opt) ini_file = os.path.join(opt["pbs"]["run_directory"], @@ -1380,6 +1414,9 @@ def makeitso(args: dict = None): f"script {all_jobs_script} like this:\n" f"bash {all_jobs_script}") + # Update number of segments for coupling in case of year rollover segments were added. + if "coupling" in args: + options["pbs"]["num_segments"] = str(len(pbs_scripts)-len(spinup_pbs_scripts)-len(warmup_pbs_scripts)) # Return the options dict used to create the PBS scripts, and the list # of PBS scripts which constitute the warmup period. return options, spinup_pbs_scripts, warmup_pbs_scripts diff --git a/scripts/makeitso/option_descriptions.json b/scripts/makeitso/option_descriptions.json index 6ea9b7c4..7d2a9dc4 100644 --- a/scripts/makeitso/option_descriptions.json +++ b/scripts/makeitso/option_descriptions.json @@ -57,8 +57,8 @@ "hpc_system": { "LEVEL": "BASIC", "prompt": "Name of HPC system", - "default": "atiken", - "valids": ["derecho", "atiken"] + "default": "aitken", + "valids": ["derecho", "aitken"] } }, @@ -181,7 +181,7 @@ "default": "mpiexec pinCpuCores.sh" } }, - "atiken": { + "aitken": { "group_list": { "LEVEL": "BASIC", "prompt": "PBS group list", @@ -488,7 +488,7 @@ "prompt": "(GAMERA) Block halo MPI", "default": { "derecho": "T", - "atiken": "F" + "aitken": "F" }, "valids": ["T", "F"] } @@ -574,7 +574,7 @@ "prompt": "(VOLTRON) Perform asynchronous model coupling", "default": { "derecho": "F", - "atiken": "T" + "aitken": "T" }, "valids": ["T", "F"] } diff --git a/scripts/makeitso/option_engage_descriptions.json b/scripts/makeitso/option_engage_descriptions.json index 8b9f98c6..49bf4faf 100644 --- a/scripts/makeitso/option_engage_descriptions.json +++ b/scripts/makeitso/option_engage_descriptions.json @@ -54,7 +54,7 @@ ] } }, - "atiken": { + "aitken": { "modules": { "LEVEL": "EXPERT", "prompt": "Modules to load", @@ -78,7 +78,7 @@ "LEVEL": "EXPERT", "prompt": "Commands to add local module PATH", "default": [ - "export PREFIX=/u/nrao3/local_atiken", + "export PREFIX=/u/nrao3/local_aitken", "export LIBRARY_PATH=${LIBRARY_PATH}:$PREFIX/lib", "export LD_LIBRARY_PATH=$LIBRARY_PATH", "export CPATH=$PREFIX/include",