Merged in dev_atiken_update (pull request #42)

Aitken update + year boundary fix

Approved-by: Jeff
Approved-by: Slava Merkin

scripts/makeitso-gamhelio/option_descriptions.json

@@ -35,8 +35,8 @@
         "hpc_system": {
             "LEVEL": "BASIC",
             "prompt": "Name of HPC system",
-            "default": "pleiades",
-            "valids": ["derecho", "pleiades"]
+            "default": "aitken",
+            "valids": ["derecho", "aitken"]
         }
     },
 
@@ -129,7 +129,7 @@
                 "default": "mpiexec pinCpuCores.sh"
             }
         },
-        "pleiades": {
+        "aitken": {
             "queue": {
                 "LEVEL": "BASIC",
                 "prompt": "PBS queue name",
@@ -149,7 +149,7 @@
             "ncpus": {
                 "LEVEL": "EXPERT",
                 "prompt": "Number of cores per node",
-                "default": "28"
+                "default": "128"
             },
             "mpiprocs": {
                 "LEVEL": "EXPERT",
@@ -159,12 +159,12 @@
             "ompthreads": {
                 "LEVEL": "EXPERT",
                 "prompt": "Number of OMP threads per MPI rank",
-                "default": "14"
+                "default": "64"
             },
             "other": {
                 "LEVEL": "EXPERT",
                 "prompt": "Additional options for PBS -l",
-                "default": ":model=bro"
+                "default": ":model=mil_ait"
             },
             "modules": {
                 "LEVEL": "EXPERT",
@@ -455,7 +455,7 @@
                 "prompt": "(GAMERA) Block halo MPI",
                 "default": {
                     "derecho": "T",
-                    "pleiades": "F"
+                    "aitken": "F"
                 },
                 "valids": ["T", "F"]
             }

scripts/makeitso/engage.py

@@ -163,7 +163,7 @@ def create_pbs_scripts(gr_options: dict, makeitso_options:dict,makeitso_pbs_scri
     options["pbs"]["tie_mpiprocs"] = tiegcm_options["job"]["resource"]["mpiprocs"]
     options["pbs"]["tie_mpiranks"] = tiegcm_options["job"]["nprocs"]
     options["pbs"]["tie_exe"] = tiegcm_options["model"]["data"]["coupled_modelexe"] 
-    if tiegcm_options["simulation"]["hpc_system"] == "pleiades":
+    if tiegcm_options["simulation"]["hpc_system"] == "aitken":
         options["pbs"]["model"] = tiegcm_options["job"]["resource"]["model"]    
     
     # GR PBS parameters
@@ -181,7 +181,7 @@ def create_pbs_scripts(gr_options: dict, makeitso_options:dict,makeitso_pbs_scri
     if tiegcm_options["simulation"]["hpc_system"] == "derecho":
         options["pbs"]["mpiexec_command"] = "mpiexec"
         options["pbs"]["mpiexec_option"] = "-n"
-    elif tiegcm_options["simulation"]["hpc_system"] == "pleiades":
+    elif tiegcm_options["simulation"]["hpc_system"] == "aitken":
         options["pbs"]["mpiexec_command"] = "mpiexec_mpt"
         options["pbs"]["mpiexec_option"] = "-np"
         options["pbs"]["tie_scripts"] = "correctOMPenvironment.sh $NODEFILE_1 omplace"
@@ -221,7 +221,9 @@ echo "Running tiegcm and voltron at the same time"
     
     # Create a PBS script for each segment.
     pbs_scripts = []
+    print(f'Creating {options["pbs"]["num_segments"]} PBS scripts')
     for job in range(1,int(options["pbs"]["num_segments"])+1):
+
         opt = copy.deepcopy(options)  # Need a copy of options
         runid = opt["simulation"]["job_name"]
         segment_id = f"{runid}-{job:02d}"
@@ -402,7 +404,7 @@ def prompt_user_for_run_options(args):
         od["num_helpers"]["default"][gamera_grid_type]
     )
     od["modules"] = oed["modules"]
-    if hpc_platform == "pleiades":
+    if hpc_platform == "aitken":
         od["moduledir"] = oed["moduledir"]
         od["local_modules"] = oed["local_modules"]
     for on in od:
@@ -495,7 +497,7 @@ def main():
             od["num_helpers"]["default"][gamera_grid_type]
         )
         od["modules"] = oed["modules"]
-        if hpc_platform == "pleiades":
+        if hpc_platform == "aitken":
             od["moduledir"] = oed["moduledir"]
             od["local_modules"] = oed["local_modules"]
         for on in od:
@@ -627,6 +629,8 @@ def main():
     with open('makeitso_parameters.json', 'w') as f:
         json.dump(makeitso_options, f, indent=JSON_INDENT)
     
+    # Update engage options with makeitso options
+    engage_options["pbs"]["num_segments"] = makeitso_options["pbs"]["num_segments"]
 
     # Run the TIEGCMrun
     coupled_options = copy.deepcopy(engage_options)

scripts/makeitso/makeitso.py

@@ -254,7 +254,7 @@ def update_option_descriptions(option_descriptions: dict, args: dict):
                 od["default"] = pbs[k]
         # Incorporate HPC platform-specific PBS options from engage in default.
         option_descriptions["pbs"][hpc_platform]["modules"]["default"] = pbs["modules"]
-        if hpc_platform == "pleiades":
+        if hpc_platform == "aitken":
             option_descriptions["pbs"][hpc_platform]["moduledir"]["default"] = pbs["moduledir"]
             option_descriptions["pbs"][hpc_platform]["local_modules"]["default"] = pbs["local_modules"]
     # Return the option descriptions.
@@ -987,10 +987,29 @@ def create_ini_files(options: dict, args: dict):
             num_warmup_segments = i_last_warmup_ini 
         # Create an .ini file for each simulation segment. Files for each
         # segment will be numbered starting with 1.
+        start_dt = datetime.datetime.fromisoformat(options["simulation"]["start_date"])
         if "coupling" in args:
             num_segments = math.ceil((simulation_duration - num_warmup_segments*warmup_segment_duration)/segment_duration)
+            for job in range(1, num_segments + 1):
+                dT = float(options["simulation"]["segment_duration"])
+                tfin_delta = float(coupling["gr_warm_up_time"])
+                tStart_segment = (job - 1)*dT + tfin_delta
+                tFin_segment = job*dT + tfin_delta + 1.0
+                # Check if the segment end time is exactly Jan 1, 00:00:00 of any year 
+                # (Helps with year boundary issues in TIEGCM).
+                # If so, adjust the segment end time to be Jan 1 of the next year.
+                # This is only done if the segment start and end times are in different years.
+                segment_start_dt = start_dt + datetime.timedelta(seconds=tStart_segment)
+                segment_end_dt = start_dt + datetime.timedelta(seconds=tFin_segment-1)
+                if (segment_start_dt).year != (segment_end_dt).year:
+                    if segment_end_dt.month != 1 or segment_end_dt.day != 1 or segment_end_dt.hour != 0 or segment_end_dt.minute != 0 or segment_end_dt.second != 0:
+                        num_segments += 1                       
+                        break
+
         else:
             num_segments = int(options["pbs"]["num_segments"])
+        job_offset = 0
+        tFin_offset = 0
         for job in range(1, num_segments + 1):
             opt = copy.deepcopy(options)  # Need a copy of options
             runid = opt["simulation"]["job_name"]
@@ -1007,7 +1026,22 @@ def create_ini_files(options: dict, args: dict):
             else:
                 tfin_delta = 0.0
             # Add 1 to ensure last restart file created    
-            tFin_segment = job*dT + tfin_delta + 1.0
+            tStart_segment = (job - 1)*dT + tfin_delta + tFin_offset
+            
+            tFin_segment = (job - job_offset)*dT + tfin_delta + 1.0
+
+            segment_start_dt = start_dt + datetime.timedelta(seconds=tStart_segment)
+            segment_end_dt = start_dt + datetime.timedelta(seconds=tFin_segment-1)
+            if "coupling" in args and segment_start_dt.year != segment_end_dt.year: 
+                if segment_end_dt.month != 1 or segment_end_dt.day != 1 or segment_end_dt.hour != 0 or segment_end_dt.minute != 0 or segment_end_dt.second != 0:
+                    print(f'Segment {job} crosses year boundary, adjusting segment end time to Jan 1 of next year.')
+                    next_year = segment_start_dt.year + 1
+                    jan1_next_year = datetime.datetime(next_year, 1, 1, 0, 0, 0)
+                    tFin_segment = (jan1_next_year - start_dt).total_seconds() + 1.0
+                    tFin_offset = tFin_segment - (job - job_offset)*dT - tfin_delta - 1.0
+                    job_offset = 1
+                    segment_end_dt = start_dt + datetime.timedelta(seconds=tFin_segment - 1.0)
+
             nRes = int(((tFin_segment - 1) - dT )/dtRes) 
             opt["gamera"]["restart"]["nRes"] = str(nRes)   
             # Engage modifications to parameters in coupled segment.
@@ -1019,7 +1053,9 @@ def create_ini_files(options: dict, args: dict):
                 else:
                     # Subtract 1 from tFin padding for coupling beacuse to offset the +1.0 for restart file done above.
                     tfin_padding = tfin_coupling_padding - 1.0
-            opt["voltron"]["time"]["tFin"] = str(tFin_segment + tfin_padding)
+            
+            tFin = tFin_segment + tfin_padding
+            opt["voltron"]["time"]["tFin"] = str(tFin)
             #print(f'Creating job {job} with tFin_seg = {opt["voltron"]["time"]["tFin"]}')
             ini_content = template.render(opt)
             ini_file = os.path.join(opt["pbs"]["run_directory"],
@@ -1380,6 +1416,9 @@ def makeitso(args: dict = None):
           f"script {all_jobs_script} like this:\n"
           f"bash {all_jobs_script}")
 
+    # Update number of segments for coupling in case of year rollover segments were added.
+    if "coupling" in args:
+        options["pbs"]["num_segments"] = str(len(pbs_scripts)-len(spinup_pbs_scripts)-len(warmup_pbs_scripts))
     # Return the options dict used to create the PBS scripts, and the list
     # of PBS scripts which constitute the warmup period.
     return options, spinup_pbs_scripts, warmup_pbs_scripts

scripts/makeitso/option_descriptions.json

@@ -57,8 +57,8 @@
         "hpc_system": {
             "LEVEL": "BASIC",
             "prompt": "Name of HPC system",
-            "default": "pleiades",
-            "valids": ["derecho", "pleiades"]
+            "default": "aitken",
+            "valids": ["derecho", "aitken"]
         }
     },
 
@@ -181,7 +181,7 @@
                 "default": "mpiexec pinCpuCores.sh"
             }
         },
-        "pleiades": {
+        "aitken": {
             "group_list": {
                 "LEVEL": "BASIC",
                 "prompt": "PBS group list",
@@ -211,7 +211,7 @@
             "ncpus": {
                 "LEVEL": "EXPERT",
                 "prompt": "Number of cores per node",
-                "default": "28"
+                "default": "128"
             },
             "mpiprocs": {
                 "LEVEL": "EXPERT",
@@ -221,7 +221,7 @@
             "ompthreads": {
                 "LEVEL": "EXPERT",
                 "prompt": "Number of OMP threads per MPI rank",
-                "default": "14"
+                "default": "64"
             },
             "num_helpers": {
                 "LEVEL": "EXPERT",
@@ -241,12 +241,12 @@
             "helper_ompthreads": {
                 "LEVEL": "EXPERT",
                 "prompt": "Number of OMP threads per helper node",
-                "default": "28"
+                "default": "128"
             },
             "other": {
                 "LEVEL": "EXPERT",
                 "prompt": "Additional options for PBS -l",
-                "default": ":model=bro"
+                "default": ":model=mil_ait"
             },
             "modules": {
                 "LEVEL": "EXPERT",
@@ -488,7 +488,7 @@
                 "prompt": "(GAMERA) Block halo MPI",
                 "default": {
                     "derecho": "T",
-                    "pleiades": "F"
+                    "aitken": "F"
                 },
                 "valids": ["T", "F"]
             }
@@ -574,7 +574,7 @@
                 "prompt": "(VOLTRON) Perform asynchronous model coupling",
                 "default": {
                     "derecho": "F",
-                    "pleiades": "T"
+                    "aitken": "T"
                 },
                 "valids": ["T", "F"]
             }

scripts/makeitso/option_engage_descriptions.json

@@ -54,7 +54,7 @@
                 ]
             }
         },
-        "pleiades": {
+        "aitken": {
             "modules": {
                 "LEVEL": "EXPERT",
                 "prompt": "Modules to load",
@@ -78,7 +78,7 @@
                 "LEVEL": "EXPERT",
                 "prompt": "Commands to add local module PATH",
                 "default": [
-                    "export PREFIX=/u/nrao3/local3",
+                    "export PREFIX=/u/nrao3/local_aitken",
                     "export LIBRARY_PATH=${LIBRARY_PATH}:$PREFIX/lib",
                     "export LD_LIBRARY_PATH=$LIBRARY_PATH",
                     "export CPATH=$PREFIX/include",