mirror of
https://github.com/JHUAPL/kaiju.git
synced 2026-01-08 22:58:05 -05:00
Merge remote-tracking branch 'origin/development' into ewinter-derecho_testing
This commit is contained in:
Eric Winter
3
.gitmodules
vendored
3
.gitmodules
vendored
@@ -1,3 +0,0 @@
|
||||
[submodule "wiki"]
|
||||
path = wiki
|
||||
url = https://bitbucket.org/aplkaiju/kaiju/wiki
|
||||
5
.mailmap
5
.mailmap
@@ -1,5 +0,0 @@
|
||||
Kareem <kareem.sorathia@gmail.com>
|
||||
Kareem <kareem.sorathia@gmail.com> <soratka1@jhuapl.edu>
|
||||
Jeffrey Garretson <Jeffrey.Garretson@jhuapl.edu> <jeffreyg@cheyenne1.ib0.cheyenne.ucar.edu>
|
||||
Jeffrey Garretson <Jeffrey.Garretson@jhuapl.edu> <jeffreyg@cheyenne3.ib0.cheyenne.ucar.edu>
|
||||
Jeffrey Garretson <Jeffrey.Garretson@jhuapl.edu> <jeffreyg@cheyenne4.ib0.cheyenne.ucar.edu>
|
||||
@@ -35,8 +35,8 @@
|
||||
"hpc_system": {
|
||||
"LEVEL": "BASIC",
|
||||
"prompt": "Name of HPC system",
|
||||
"default": "pleiades",
|
||||
"valids": ["derecho", "pleiades"]
|
||||
"default": "aitken",
|
||||
"valids": ["derecho", "aitken"]
|
||||
}
|
||||
},
|
||||
|
||||
@@ -129,7 +129,7 @@
|
||||
"default": "mpiexec pinCpuCores.sh"
|
||||
}
|
||||
},
|
||||
"pleiades": {
|
||||
"aitken": {
|
||||
"queue": {
|
||||
"LEVEL": "BASIC",
|
||||
"prompt": "PBS queue name",
|
||||
@@ -149,7 +149,7 @@
|
||||
"ncpus": {
|
||||
"LEVEL": "EXPERT",
|
||||
"prompt": "Number of cores per node",
|
||||
"default": "28"
|
||||
"default": "128"
|
||||
},
|
||||
"mpiprocs": {
|
||||
"LEVEL": "EXPERT",
|
||||
@@ -159,12 +159,12 @@
|
||||
"ompthreads": {
|
||||
"LEVEL": "EXPERT",
|
||||
"prompt": "Number of OMP threads per MPI rank",
|
||||
"default": "14"
|
||||
"default": "64"
|
||||
},
|
||||
"other": {
|
||||
"LEVEL": "EXPERT",
|
||||
"prompt": "Additional options for PBS -l",
|
||||
"default": ":model=bro"
|
||||
"default": ":model=mil_ait"
|
||||
},
|
||||
"modules": {
|
||||
"LEVEL": "EXPERT",
|
||||
@@ -455,7 +455,7 @@
|
||||
"prompt": "(GAMERA) Block halo MPI",
|
||||
"default": {
|
||||
"derecho": "T",
|
||||
"pleiades": "F"
|
||||
"aitken": "F"
|
||||
},
|
||||
"valids": ["T", "F"]
|
||||
}
|
||||
|
||||
@@ -163,7 +163,7 @@ def create_pbs_scripts(gr_options: dict, makeitso_options:dict,makeitso_pbs_scri
|
||||
options["pbs"]["tie_mpiprocs"] = tiegcm_options["job"]["resource"]["mpiprocs"]
|
||||
options["pbs"]["tie_mpiranks"] = tiegcm_options["job"]["nprocs"]
|
||||
options["pbs"]["tie_exe"] = tiegcm_options["model"]["data"]["coupled_modelexe"]
|
||||
if tiegcm_options["simulation"]["hpc_system"] == "pleiades":
|
||||
if tiegcm_options["simulation"]["hpc_system"] == "aitken":
|
||||
options["pbs"]["model"] = tiegcm_options["job"]["resource"]["model"]
|
||||
|
||||
# GR PBS parameters
|
||||
@@ -181,7 +181,7 @@ def create_pbs_scripts(gr_options: dict, makeitso_options:dict,makeitso_pbs_scri
|
||||
if tiegcm_options["simulation"]["hpc_system"] == "derecho":
|
||||
options["pbs"]["mpiexec_command"] = "mpiexec"
|
||||
options["pbs"]["mpiexec_option"] = "-n"
|
||||
elif tiegcm_options["simulation"]["hpc_system"] == "pleiades":
|
||||
elif tiegcm_options["simulation"]["hpc_system"] == "aitken":
|
||||
options["pbs"]["mpiexec_command"] = "mpiexec_mpt"
|
||||
options["pbs"]["mpiexec_option"] = "-np"
|
||||
options["pbs"]["tie_scripts"] = "correctOMPenvironment.sh $NODEFILE_1 omplace"
|
||||
@@ -221,7 +221,9 @@ echo "Running tiegcm and voltron at the same time"
|
||||
|
||||
# Create a PBS script for each segment.
|
||||
pbs_scripts = []
|
||||
print(f'Creating {options["pbs"]["num_segments"]} PBS scripts')
|
||||
for job in range(1,int(options["pbs"]["num_segments"])+1):
|
||||
|
||||
opt = copy.deepcopy(options) # Need a copy of options
|
||||
runid = opt["simulation"]["job_name"]
|
||||
segment_id = f"{runid}-{job:02d}"
|
||||
@@ -402,7 +404,7 @@ def prompt_user_for_run_options(args):
|
||||
od["num_helpers"]["default"][gamera_grid_type]
|
||||
)
|
||||
od["modules"] = oed["modules"]
|
||||
if hpc_platform == "pleiades":
|
||||
if hpc_platform == "aitken":
|
||||
od["moduledir"] = oed["moduledir"]
|
||||
od["local_modules"] = oed["local_modules"]
|
||||
for on in od:
|
||||
@@ -495,7 +497,7 @@ def main():
|
||||
od["num_helpers"]["default"][gamera_grid_type]
|
||||
)
|
||||
od["modules"] = oed["modules"]
|
||||
if hpc_platform == "pleiades":
|
||||
if hpc_platform == "aitken":
|
||||
od["moduledir"] = oed["moduledir"]
|
||||
od["local_modules"] = oed["local_modules"]
|
||||
for on in od:
|
||||
@@ -627,6 +629,8 @@ def main():
|
||||
with open('makeitso_parameters.json', 'w') as f:
|
||||
json.dump(makeitso_options, f, indent=JSON_INDENT)
|
||||
|
||||
# Update engage options with makeitso options
|
||||
engage_options["pbs"]["num_segments"] = makeitso_options["pbs"]["num_segments"]
|
||||
|
||||
# Run the TIEGCMrun
|
||||
coupled_options = copy.deepcopy(engage_options)
|
||||
|
||||
@@ -254,7 +254,7 @@ def update_option_descriptions(option_descriptions: dict, args: dict):
|
||||
od["default"] = pbs[k]
|
||||
# Incorporate HPC platform-specific PBS options from engage in default.
|
||||
option_descriptions["pbs"][hpc_platform]["modules"]["default"] = pbs["modules"]
|
||||
if hpc_platform == "pleiades":
|
||||
if hpc_platform == "aitken":
|
||||
option_descriptions["pbs"][hpc_platform]["moduledir"]["default"] = pbs["moduledir"]
|
||||
option_descriptions["pbs"][hpc_platform]["local_modules"]["default"] = pbs["local_modules"]
|
||||
# Return the option descriptions.
|
||||
@@ -987,10 +987,29 @@ def create_ini_files(options: dict, args: dict):
|
||||
num_warmup_segments = i_last_warmup_ini
|
||||
# Create an .ini file for each simulation segment. Files for each
|
||||
# segment will be numbered starting with 1.
|
||||
start_dt = datetime.datetime.fromisoformat(options["simulation"]["start_date"])
|
||||
if "coupling" in args:
|
||||
num_segments = math.ceil((simulation_duration - num_warmup_segments*warmup_segment_duration)/segment_duration)
|
||||
for job in range(1, num_segments + 1):
|
||||
dT = float(options["simulation"]["segment_duration"])
|
||||
tfin_delta = float(coupling["gr_warm_up_time"])
|
||||
tStart_segment = (job - 1)*dT + tfin_delta
|
||||
tFin_segment = job*dT + tfin_delta + 1.0
|
||||
# Check if the segment end time is exactly Jan 1, 00:00:00 of any year
|
||||
# (Helps with year boundary issues in TIEGCM).
|
||||
# If so, adjust the segment end time to be Jan 1 of the next year.
|
||||
# This is only done if the segment start and end times are in different years.
|
||||
segment_start_dt = start_dt + datetime.timedelta(seconds=tStart_segment)
|
||||
segment_end_dt = start_dt + datetime.timedelta(seconds=tFin_segment-1)
|
||||
if (segment_start_dt).year != (segment_end_dt).year:
|
||||
if segment_end_dt.month != 1 or segment_end_dt.day != 1 or segment_end_dt.hour != 0 or segment_end_dt.minute != 0 or segment_end_dt.second != 0:
|
||||
num_segments += 1
|
||||
break
|
||||
|
||||
else:
|
||||
num_segments = int(options["pbs"]["num_segments"])
|
||||
job_offset = 0
|
||||
tFin_offset = 0
|
||||
for job in range(1, num_segments + 1):
|
||||
opt = copy.deepcopy(options) # Need a copy of options
|
||||
runid = opt["simulation"]["job_name"]
|
||||
@@ -1007,7 +1026,22 @@ def create_ini_files(options: dict, args: dict):
|
||||
else:
|
||||
tfin_delta = 0.0
|
||||
# Add 1 to ensure last restart file created
|
||||
tFin_segment = job*dT + tfin_delta + 1.0
|
||||
tStart_segment = (job - 1)*dT + tfin_delta + tFin_offset
|
||||
|
||||
tFin_segment = (job - job_offset)*dT + tfin_delta + 1.0
|
||||
|
||||
segment_start_dt = start_dt + datetime.timedelta(seconds=tStart_segment)
|
||||
segment_end_dt = start_dt + datetime.timedelta(seconds=tFin_segment-1)
|
||||
if "coupling" in args and segment_start_dt.year != segment_end_dt.year:
|
||||
if segment_end_dt.month != 1 or segment_end_dt.day != 1 or segment_end_dt.hour != 0 or segment_end_dt.minute != 0 or segment_end_dt.second != 0:
|
||||
print(f'Segment {job} crosses year boundary, adjusting segment end time to Jan 1 of next year.')
|
||||
next_year = segment_start_dt.year + 1
|
||||
jan1_next_year = datetime.datetime(next_year, 1, 1, 0, 0, 0)
|
||||
tFin_segment = (jan1_next_year - start_dt).total_seconds() + 1.0
|
||||
tFin_offset = tFin_segment - (job - job_offset)*dT - tfin_delta - 1.0
|
||||
job_offset = 1
|
||||
segment_end_dt = start_dt + datetime.timedelta(seconds=tFin_segment - 1.0)
|
||||
|
||||
nRes = int(((tFin_segment - 1) - dT )/dtRes)
|
||||
opt["gamera"]["restart"]["nRes"] = str(nRes)
|
||||
# Engage modifications to parameters in coupled segment.
|
||||
@@ -1019,7 +1053,9 @@ def create_ini_files(options: dict, args: dict):
|
||||
else:
|
||||
# Subtract 1 from tFin padding for coupling beacuse to offset the +1.0 for restart file done above.
|
||||
tfin_padding = tfin_coupling_padding - 1.0
|
||||
opt["voltron"]["time"]["tFin"] = str(tFin_segment + tfin_padding)
|
||||
|
||||
tFin = tFin_segment + tfin_padding
|
||||
opt["voltron"]["time"]["tFin"] = str(tFin)
|
||||
#print(f'Creating job {job} with tFin_seg = {opt["voltron"]["time"]["tFin"]}')
|
||||
ini_content = template.render(opt)
|
||||
ini_file = os.path.join(opt["pbs"]["run_directory"],
|
||||
@@ -1380,6 +1416,9 @@ def makeitso(args: dict = None):
|
||||
f"script {all_jobs_script} like this:\n"
|
||||
f"bash {all_jobs_script}")
|
||||
|
||||
# Update number of segments for coupling in case of year rollover segments were added.
|
||||
if "coupling" in args:
|
||||
options["pbs"]["num_segments"] = str(len(pbs_scripts)-len(spinup_pbs_scripts)-len(warmup_pbs_scripts))
|
||||
# Return the options dict used to create the PBS scripts, and the list
|
||||
# of PBS scripts which constitute the warmup period.
|
||||
return options, spinup_pbs_scripts, warmup_pbs_scripts
|
||||
|
||||
@@ -57,8 +57,8 @@
|
||||
"hpc_system": {
|
||||
"LEVEL": "BASIC",
|
||||
"prompt": "Name of HPC system",
|
||||
"default": "pleiades",
|
||||
"valids": ["derecho", "pleiades"]
|
||||
"default": "aitken",
|
||||
"valids": ["derecho", "aitken"]
|
||||
}
|
||||
},
|
||||
|
||||
@@ -181,7 +181,7 @@
|
||||
"default": "mpiexec pinCpuCores.sh"
|
||||
}
|
||||
},
|
||||
"pleiades": {
|
||||
"aitken": {
|
||||
"group_list": {
|
||||
"LEVEL": "BASIC",
|
||||
"prompt": "PBS group list",
|
||||
@@ -211,7 +211,7 @@
|
||||
"ncpus": {
|
||||
"LEVEL": "EXPERT",
|
||||
"prompt": "Number of cores per node",
|
||||
"default": "28"
|
||||
"default": "128"
|
||||
},
|
||||
"mpiprocs": {
|
||||
"LEVEL": "EXPERT",
|
||||
@@ -221,7 +221,7 @@
|
||||
"ompthreads": {
|
||||
"LEVEL": "EXPERT",
|
||||
"prompt": "Number of OMP threads per MPI rank",
|
||||
"default": "14"
|
||||
"default": "64"
|
||||
},
|
||||
"num_helpers": {
|
||||
"LEVEL": "EXPERT",
|
||||
@@ -241,12 +241,12 @@
|
||||
"helper_ompthreads": {
|
||||
"LEVEL": "EXPERT",
|
||||
"prompt": "Number of OMP threads per helper node",
|
||||
"default": "28"
|
||||
"default": "128"
|
||||
},
|
||||
"other": {
|
||||
"LEVEL": "EXPERT",
|
||||
"prompt": "Additional options for PBS -l",
|
||||
"default": ":model=bro"
|
||||
"default": ":model=mil_ait"
|
||||
},
|
||||
"modules": {
|
||||
"LEVEL": "EXPERT",
|
||||
@@ -488,7 +488,7 @@
|
||||
"prompt": "(GAMERA) Block halo MPI",
|
||||
"default": {
|
||||
"derecho": "T",
|
||||
"pleiades": "F"
|
||||
"aitken": "F"
|
||||
},
|
||||
"valids": ["T", "F"]
|
||||
}
|
||||
@@ -574,7 +574,7 @@
|
||||
"prompt": "(VOLTRON) Perform asynchronous model coupling",
|
||||
"default": {
|
||||
"derecho": "F",
|
||||
"pleiades": "T"
|
||||
"aitken": "T"
|
||||
},
|
||||
"valids": ["T", "F"]
|
||||
}
|
||||
|
||||
@@ -54,7 +54,7 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"pleiades": {
|
||||
"aitken": {
|
||||
"modules": {
|
||||
"LEVEL": "EXPERT",
|
||||
"prompt": "Modules to load",
|
||||
@@ -78,7 +78,7 @@
|
||||
"LEVEL": "EXPERT",
|
||||
"prompt": "Commands to add local module PATH",
|
||||
"default": [
|
||||
"export PREFIX=/u/nrao3/local3",
|
||||
"export PREFIX=/u/nrao3/local_aitken",
|
||||
"export LIBRARY_PATH=${LIBRARY_PATH}:$PREFIX/lib",
|
||||
"export LD_LIBRARY_PATH=$LIBRARY_PATH",
|
||||
"export CPATH=$PREFIX/include",
|
||||
|
||||
@@ -16,6 +16,8 @@ module chopio
|
||||
integer, parameter :: MAXEBVS = 50
|
||||
|
||||
integer :: Nx1 = 64, Nx2 = 64, Nx3 = 64
|
||||
real(rp) :: originOff = 0.0 !Default origin offset
|
||||
real(rp) :: lookDir = 0.0 !Default CONE grid tilt
|
||||
|
||||
real(rp) :: xyzMax = 10.0 !Default bound
|
||||
real(rp), dimension(:,:,:), allocatable :: xxi,yyi,zzi,xxc,yyc,zzc
|
||||
@@ -37,6 +39,10 @@ module chopio
|
||||
integer :: iS,iE
|
||||
real(rp) :: xcc(NDIM)
|
||||
logical :: doLogR
|
||||
real(rp) :: xOff,yOff,zOff !Cartesian offsets for CONE grid origin
|
||||
real(rp) :: theta_pitch,phi_yaw !Theta and phi for y and z rotations in 3D space
|
||||
real(rp) :: xxi_temp,yyi_temp,zzi_temp !Temp variables for 3D rotations
|
||||
real(rp) :: xxi_tilt,yyi_tilt,zzi_tilt,xxi_pan,yyi_pan,zzi_pan !More temp variables for 3D rotations
|
||||
|
||||
!Check for time parallelism
|
||||
call InitParInTime(Model,inpXML,"eb3",eb3DOutF)
|
||||
@@ -61,6 +67,15 @@ module chopio
|
||||
call inpXML%Set_Val(Nx2,'chop/Nx2',Nx2)
|
||||
call inpXML%Set_Val(Nx3,'chop/Nx3',Nx3)
|
||||
|
||||
!Get origin offset for CONE chopping. Default to originOff (zero)
|
||||
call inpXML%Set_Val(xOff,'chop/xOff',originOff)
|
||||
call inpXML%Set_Val(yOff,'chop/yOff',originOff)
|
||||
call inpXML%Set_Val(zOff,'chop/zOff',originOff)
|
||||
|
||||
!Get pitch and yaw angles for CONE chopping. Default to lookDir (zero)
|
||||
call inpXML%Set_Val(theta_pitch,'chop/lookPol',lookDir)
|
||||
call inpXML%Set_Val(phi_yaw,'chop/lookAzi',lookDir)
|
||||
|
||||
!for LFM grid, take all i-shells within x1Max, in the Sun direction
|
||||
select case(trim(toUpper(idStr)))
|
||||
!---------
|
||||
@@ -120,11 +135,12 @@ module chopio
|
||||
endif
|
||||
dx2 = (x2Max-x2Min)*deg2rad/Nx2
|
||||
dx3 = (x3Max-x3Min)*deg2rad/Nx3
|
||||
!$OMP PARALLEL DO
|
||||
!$OMP PARALLEL DO default(shared) &
|
||||
!$OMP private(i,j,k,x1,x2,x3)
|
||||
do k=1, Nx3+1
|
||||
do j=1,Nx2+1
|
||||
do i=1,Nx1+1
|
||||
!x1 = Rin + (i-1)*dx1
|
||||
!x1 = Rin(i,j,k) + (i-1)*dx1
|
||||
if (doLogR) then
|
||||
x1 = 10**( log10(x1Min) + (i-1)*dx1 )
|
||||
else
|
||||
@@ -141,6 +157,57 @@ module chopio
|
||||
enddo
|
||||
enddo
|
||||
!---------
|
||||
! Conical/Solid angle (based on RTP)
|
||||
case("CONE")
|
||||
!Check for log spacing in r
|
||||
call inpXML%Set_Val(doLogR,'chop/doLogR',.false.)
|
||||
if (doLogR) then
|
||||
dx1 = ( log10(x1Max)-log10(x1Min) )/Nx1
|
||||
else
|
||||
dx1 = (x1Max - x1Min)/Nx1
|
||||
endif
|
||||
dx2 = (x2Max-x2Min)*deg2rad/Nx2
|
||||
dx3 = (x3Max-x3Min)*deg2rad/Nx3
|
||||
!$OMP PARALLEL DO default(shared) &
|
||||
!$OMP private(i,j,k,x1,x2,x3,xxi_temp,yyi_temp,zzi_temp,xxi_tilt,yyi_tilt,zzi_tilt,xxi_pan,yyi_pan,zzi_pan)
|
||||
do k=1,Nx3+1
|
||||
do j=1,Nx2+1
|
||||
do i=1,Nx1+1
|
||||
!x1 = Rin(i,j,k) + (i-1)*dx1
|
||||
if (doLogR) then
|
||||
x1 = 10**( log10(x1Min) + (i-1)*dx1 )
|
||||
else
|
||||
x1 = x1Min + (i-1)*dx1
|
||||
endif
|
||||
x2 = x2Min*deg2rad + (j-1)*dx2 !Theta
|
||||
x3 = x3Min*deg2rad + (k-1)*dx3 !Phi
|
||||
|
||||
! pre-rotation variables
|
||||
xxi_temp = x1*cos(x3)*sin(x2)
|
||||
yyi_temp = x1*sin(x3)*sin(x2)
|
||||
zzi_temp = x1*cos(x2)
|
||||
|
||||
! first tilt (polar) then pan (azimuthal) (pitch then yaw)
|
||||
! perform rotation about y axis (theta, polar angle)
|
||||
xxi_tilt = (cos(theta_pitch*deg2rad)*xxi_temp) + (0) + (sin(theta_pitch*deg2rad)*zzi_temp)
|
||||
yyi_tilt = (0) + (yyi_temp) + (0)
|
||||
zzi_tilt = (-sin(theta_pitch*deg2rad)*xxi_temp) + (0) + (cos(theta_pitch*deg2rad)*zzi_temp)
|
||||
|
||||
! perform rotation about z axis (phi, azimuthal angle)
|
||||
! apply to the tilted xxi/yyi/zzi
|
||||
xxi_pan = (cos(phi_yaw*deg2rad)*xxi_tilt) + (-sin(phi_yaw*deg2rad)*yyi_tilt) + (0)
|
||||
yyi_pan = (sin(phi_yaw*deg2rad)*xxi_tilt) + (cos(phi_yaw*deg2rad)*yyi_tilt) + (0)
|
||||
zzi_pan = (0) + (0) + (zzi_tilt)
|
||||
|
||||
! post-rotation, add cartesian offsets
|
||||
xxi(i,j,k) = xxi_pan + xOff
|
||||
yyi(i,j,k) = yyi_pan + yOff
|
||||
zzi(i,j,k) = zzi_pan + zOff
|
||||
|
||||
enddo
|
||||
enddo
|
||||
enddo
|
||||
!---------
|
||||
! only x1Max is used to set how far downtail you chop
|
||||
case("LFM")
|
||||
!$OMP PARALLEL DO
|
||||
|
||||
Reference in New Issue
Block a user