From b8cf636d7c050957a40c67ea4d85d4c64cc6e04f Mon Sep 17 00:00:00 2001 From: AnonNick Date: Tue, 12 Aug 2025 09:59:58 -0700 Subject: [PATCH 01/11] Engage split bugfix --- scripts/makeitso/engage.py | 11 +++++++++-- scripts/makeitso/makeitso.py | 33 ++++++++++++++++++++++++++------- 2 files changed, 35 insertions(+), 9 deletions(-) mode change 100755 => 100644 scripts/makeitso/makeitso.py diff --git a/scripts/makeitso/engage.py b/scripts/makeitso/engage.py index d038c774..b64d3e80 100755 --- a/scripts/makeitso/engage.py +++ b/scripts/makeitso/engage.py @@ -564,7 +564,12 @@ def main(): segment_duration = float(engage_options["simulation"]["segment_duration"]) makeitso_options["voltron"]["time"]["tFin"] = int((t1-t0).total_seconds()) - makeitso_options["pbs"]["num_segments"] = str(int((t1-t0).total_seconds()/segment_duration)) + num_segments = (t1-t0).total_seconds()/segment_duration + if num_segments > int(num_segments): + num_segments = int(num_segments) + 1 + else: + num_segments = int(num_segments) + makeitso_options["pbs"]["num_segments"] = str(num_segments) select2 = 1 + int(makeitso_options["pbs"]["num_helpers"]) makeitso_options["pbs"]["select2"] = str(select2) @@ -652,7 +657,9 @@ def main(): # Create the PBS job scripts. pbs_scripts, submit_all_jobs_script = create_pbs_scripts(engage_options,makeitso_options, makeitso_pbs_scripts, tiegcm_options, tiegcm_inp_scripts, tiegcm_pbs_scripts) - print(f"pbs_scripts = {pbs_scripts}") + print(f"GR_pbs_scripts = {makeitso_pbs_scripts}") + print(f"Tiegcm_pbs_scripts = {tiegcm_pbs_scripts}") + print(f"GTR_pbs_scripts = {pbs_scripts}") print(f"submit_all_jobs_script = {submit_all_jobs_script}") diff --git a/scripts/makeitso/makeitso.py b/scripts/makeitso/makeitso.py old mode 100755 new mode 100644 index b9aeb886..74d46e98 --- a/scripts/makeitso/makeitso.py +++ b/scripts/makeitso/makeitso.py @@ -419,7 +419,7 @@ def prompt_user_for_run_options(option_descriptions: dict, args: dict): # condition file can be generated. for on in ["bcwind_available"]: o[on] = get_run_option(on, od[on], mode) - if o["bcwind_available"] == "Y": + if o["bcwind_available"].upper() == "Y": for on in ["bcwind_file"]: o[on] = get_run_option(on, od[on], mode) # Fetch the start and stop date from the bcwind file. @@ -899,6 +899,7 @@ def create_ini_files(options: dict, args: dict): coupling = args["coupling"] gr_warm_up_time = float(coupling["gr_warm_up_time"]) segment_duration = float(options["simulation"]["segment_duration"]) + simulation_duration = float(options["voltron"]["time"]["tFin"]) i_last_warmup_ini = (gr_warm_up_time/segment_duration) if i_last_warmup_ini == int(i_last_warmup_ini): warmup_segment_duration = segment_duration @@ -984,9 +985,15 @@ def create_ini_files(options: dict, args: dict): num_warmup_segments = i_last_warmup_ini # Create an .ini file for each simulation segment. Files for each # segment will be numbered starting with 1. - print(f"Creating {options['pbs']['num_segments']} segments, " - f"with {num_warmup_segments} warmup segments.") - for job in range(1, int(options["pbs"]["num_segments"]) + 1 - num_warmup_segments): + if "coupling" in args: + num_segments = (simulation_duration - num_warmup_segments*warmup_segment_duration)/segment_duration + if num_segments > int(num_segments): + num_segments = int(num_segments) + 1 + else: + num_segments = int(num_segments) + else: + num_segments = int(options["pbs"]["num_segments"]) + for job in range(1, num_segments + 1): opt = copy.deepcopy(options) # Need a copy of options runid = opt["simulation"]["job_name"] # NOTE: This naming scheme supports a maximum of 99 segments. @@ -1009,7 +1016,7 @@ def create_ini_files(options: dict, args: dict): if "coupling" in args: opt["voltron"]["coupling"]["doGCM"] = doGCM # tFin padding different for last segment. - if job == int(options["pbs"]["num_segments"]) - num_warmup_segments: + if job == num_segments: tfin_padding = -1.0 else: # Subtract 1 from tFin padding for coupling beacuse to offset the +1.0 for restart file done above. @@ -1210,9 +1217,21 @@ def create_pbs_scripts(xml_files: list, options: dict, args: dict): coupling = args["coupling"] gr_warm_up_time = float(coupling["gr_warm_up_time"]) segment_duration = float(options["simulation"]["segment_duration"]) - i_last_warmup_pbs_script = int(gr_warm_up_time/segment_duration) + simulation_duration = float(options["voltron"]["time"]["tFin"]) + i_last_warmup_ini = (gr_warm_up_time/segment_duration) + if i_last_warmup_ini == int(i_last_warmup_ini): + warmup_segment_duration = segment_duration + else: + warmup_segment_duration = gr_warm_up_time/4 + if warmup_segment_duration != int(warmup_segment_duration): + print("Error: gr_warm_up_time is not evenly divisible by 4.") + raise ValueError("Invalid gr_warm_up_time value.") + i_last_warmup_ini = (gr_warm_up_time/warmup_segment_duration) + i_last_warmup_ini = int(i_last_warmup_ini) + num_warmup_segments = i_last_warmup_ini + #i_last_warmup_pbs_script = int(gr_warm_up_time/segment_duration) spinup_pbs_scripts.append(pbs_scripts[0]) # Spinup script is first - warmup_pbs_scripts = pbs_scripts[1:i_last_warmup_pbs_script + 1] # Warmup scripts + warmup_pbs_scripts = pbs_scripts[1:num_warmup_segments + 1] # Warmup scripts # Return the paths to the PBS scripts. return pbs_scripts, submit_all_jobs_script,spinup_pbs_scripts, warmup_pbs_scripts From b863cd34a6ee0469c679318d8ceb870116f0cc83 Mon Sep 17 00:00:00 2001 From: Slava Merkin Date: Thu, 21 Aug 2025 06:51:09 -0400 Subject: [PATCH 02/11] Removing obsolete cmake package policy and making a cosmetic change to gamhelio block to ease future merges with working helio branches. --- CMakeLists.txt | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f50c8cdb..6dfc07b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,10 +1,6 @@ cmake_minimum_required(VERSION 3.20.2) project(Kaiju Fortran) -# use the new version of CMP0074, this tells cmake to use _ROOT environment -# variables when looking for packages -cmake_policy(SET CMP0074 NEW) - # add and search for pfunit (fingers crossed) list(APPEND CMAKE_PREFIX_PATH "./external") find_package(PFUNIT QUIET) @@ -161,22 +157,6 @@ add_executable(gamera.x src/drivers/gamerax.F90 ${GAMIC}) target_link_libraries(gamera.x gamlib baselib) add_dependencies(gamera gamera.x) -#------------- -#Kaiju: Gamera helio -message("Adding Gamera Helio module ...") - -#Add source -#add_subdirectory(src/gamera) - -#Print gamera helio info -#message("\tBricksize is ${bricksize}") -message("\tIC file is ${GAMHELIC}") -add_custom_target(gamhelio ALL) -message("\tAdding executable gamhelio.x") -add_executable(gamhelio.x src/drivers/gamerax.F90 ${GAMHELIC}) -target_link_libraries(gamhelio.x gamlib baselib) -add_dependencies(gamera gamhelio.x) - #------------- #Kaiju: Dragon King message("Adding dragonking module ...") @@ -238,6 +218,18 @@ add_executable(voltron.x src/drivers/voltronx.F90) target_link_libraries(voltron.x baselib voltlib gamlib dragonkinglib remixlib chimplib raijulib) add_dependencies(voltron voltron.x) +#------------- +#Kaiju: Gamera helio +message("Adding Gamera Helio module ...") +#Print gamera helio info +message("\tBricksize is ${bricksize}") +message("\tIC file is ${GAMHELIC}") +add_custom_target(gamhelio ALL) +message("\tAdding executable gamhelio.x") +add_executable(gamhelio.x src/drivers/gamerax.F90 ${GAMHELIC}) +target_link_libraries(gamhelio.x gamlib baselib) +add_dependencies(gamera gamhelio.x) + if(ENABLE_MPI) #------------- #Kaiju: Base MPI @@ -272,6 +264,7 @@ if(ENABLE_MPI) #------------- #Kaiju: Gamera Helio MPI message("Adding Gamera Helio MPI module ...") + message("\tIC file is ${GAMHELIC}") add_custom_target(gamhelio_mpi ALL) message("\tAdding executable gamhelio_mpi.x") add_executable(gamhelio_mpi.x src/drivers/gamera_mpix.F90 ${GAMHELIC}) From 6124f44a7eab2f75c14ecd4cdf9aee97afb1a90a Mon Sep 17 00:00:00 2001 From: Slava Merkin Date: Thu, 21 Aug 2025 09:23:53 -0400 Subject: [PATCH 03/11] Reverting changes in makeitso-helio that were only supposed to be made in makeitso for geospace. --- scripts/makeitso-gamhelio/option_descriptions.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/makeitso-gamhelio/option_descriptions.json b/scripts/makeitso-gamhelio/option_descriptions.json index 1bdbeb13..894834dc 100644 --- a/scripts/makeitso-gamhelio/option_descriptions.json +++ b/scripts/makeitso-gamhelio/option_descriptions.json @@ -9,18 +9,18 @@ }, "wsa_file": { "LEVEL": "BASIC", - "prompt": "Path to WSA solar wind file to use", + "prompt": "Path to WSA boundary condition file to use", "default": "wsa.fits" }, "start_date": { "LEVEL": "BASIC", "prompt": "Start date for simulation (yyyy-mm-ddThh:mm:ss)", - "default": "2001-06-01T23:00:00" + "default": "2017-08-02T19:44:23" }, "stop_date": { "LEVEL": "BASIC", "prompt": "Stop date for simulation (yyyy-mm-ddThh:mm:ss)", - "default": "2001-06-02T01:00:00" + "default": "2017-08-02T21:44:23" }, "use_segments": { "LEVEL": "BASIC", From fee6e53b2bcd8083e6e556d636bd475d5e05b429 Mon Sep 17 00:00:00 2001 From: Slava Merkin Date: Thu, 21 Aug 2025 09:25:31 -0400 Subject: [PATCH 04/11] Removing extra blank to avoid triggering a diff. --- scripts/makeitso-gamhelio/option_descriptions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/makeitso-gamhelio/option_descriptions.json b/scripts/makeitso-gamhelio/option_descriptions.json index 894834dc..fbd43003 100644 --- a/scripts/makeitso-gamhelio/option_descriptions.json +++ b/scripts/makeitso-gamhelio/option_descriptions.json @@ -20,7 +20,7 @@ "stop_date": { "LEVEL": "BASIC", "prompt": "Stop date for simulation (yyyy-mm-ddThh:mm:ss)", - "default": "2017-08-02T21:44:23" + "default": "2017-08-02T21:44:23" }, "use_segments": { "LEVEL": "BASIC", From 4cae4f396c2f69eb888332d6772bad146a3b2ed9 Mon Sep 17 00:00:00 2001 From: Slava Merkin Date: Thu, 21 Aug 2025 09:27:48 -0400 Subject: [PATCH 05/11] Removing a duplicate dtCon entry. --- scripts/makeitso-gamhelio/option_descriptions.json | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scripts/makeitso-gamhelio/option_descriptions.json b/scripts/makeitso-gamhelio/option_descriptions.json index fbd43003..06249a1c 100644 --- a/scripts/makeitso-gamhelio/option_descriptions.json +++ b/scripts/makeitso-gamhelio/option_descriptions.json @@ -383,11 +383,6 @@ "prompt": "Cadence for status updates on screen in simulated hours", "default": "5.0" }, - "dtCon": { - "LEVEL": "EXPERT", - "prompt": "Cadence for status updates on screen in simulated hours", - "default": "5.0" - }, "doTimer": { "LEVEL": "EXPERT", "prompt": "Code timing output enabled", From 406195defcde78906153997848d2b079828f737e Mon Sep 17 00:00:00 2001 From: Jeffrey Garretson Date: Thu, 21 Aug 2025 14:22:32 -0600 Subject: [PATCH 06/11] Lots of performance output for the dashboard --- src/base/clocks.F90 | 8 +++++++- src/base/types/volttypes.F90 | 1 + src/drivers/voltron_mpix.F90 | 20 ++++++++++++++++---- src/drivers/voltronx.F90 | 10 ++++++++-- src/gamera/gamapp.F90 | 4 ++-- src/gamera/gioH5.F90 | 13 ++++++++++++- src/gamera/mpi/gamapp_mpi.F90 | 10 ++++++---- src/raiju/raijuAdvancer.F90 | 12 ++++++------ src/raiju/raijuIO.F90 | 10 ++++++++-- src/voltron/mpi/voltapp_mpi.F90 | 1 + src/voltron/voltapp.F90 | 1 + src/voltron/voltio.F90 | 14 +++++++++++++- 12 files changed, 81 insertions(+), 23 deletions(-) diff --git a/src/base/clocks.F90 b/src/base/clocks.F90 index d82862cf..cafdb0e3 100644 --- a/src/base/clocks.F90 +++ b/src/base/clocks.F90 @@ -8,6 +8,7 @@ module clocks !Global clock parameters integer, parameter :: maxClocks = 80 integer :: clockRate=0,clockMax=0 + integer :: cleanClockStep=0 !Clock output (min/max depth) integer :: clkMinD = 1, clkMaxD = 5 @@ -174,9 +175,14 @@ module clocks end subroutine Toc !Reset clocks - subroutine cleanClocks() + subroutine cleanClocks(cleanStep) + integer, optional, intent(in) :: cleanStep integer :: n + ! optional, allow the user to record what model step the clocks were cleaned at + ! this can help the user track per-step timing info + if(present(cleanStep)) cleanClockStep = cleanStep + do n=1,nclk kClocks(n)%tElap = 0 ! if the clock is active, reset the tic to right now diff --git a/src/base/types/volttypes.F90 b/src/base/types/volttypes.F90 index 7b4c76bf..ed0a416b 100644 --- a/src/base/types/volttypes.F90 +++ b/src/base/types/volttypes.F90 @@ -345,6 +345,7 @@ module volttypes type (IOClock_T) :: IO logical :: isLoud = .true. !Console output logical :: writeFiles = .true. !File output + logical :: writePerf = .true. ! Write performance data to files !Apps type(mixApp_T) :: remixApp diff --git a/src/drivers/voltron_mpix.F90 b/src/drivers/voltron_mpix.F90 index f7fb31a5..aa9fbddf 100644 --- a/src/drivers/voltron_mpix.F90 +++ b/src/drivers/voltron_mpix.F90 @@ -30,6 +30,7 @@ program voltron_mpix type(XML_Input_T) :: xmlInp real(rp) :: nextDT integer :: divideSize,i + logical :: doResetClocks = .false. ! initialize MPI !Set up MPI with or without thread support @@ -206,12 +207,12 @@ program voltron_mpix if (vApp%IO%doTimerOut) then call printClocks() endif - call cleanClocks() + doResetClocks = .true. elseif (vApp%IO%doTimer(vApp%time)) then if (vApp%IO%doTimerOut) then call printClocks() endif - call cleanClocks() + doResetClocks = .true. endif !Data output @@ -223,6 +224,12 @@ program voltron_mpix if (vApp%IO%doRestart(vApp%time)) then call resOutputV(vApp,vApp%gApp) endif + + !Reset clocks last so data is available for all output + if (doResetClocks) then + call cleanClocks(vApp%ts) + doResetClocks = .false. + endif call Toc("IO", .true.) call Toc("Omega", .true.) @@ -257,11 +264,11 @@ program voltron_mpix !Timing info if (gApp%Model%IO%doTimerOut) call printClocks() - call cleanClocks() + doResetClocks = .true. elseif (gApp%Model%IO%doTimer(gApp%Model%t)) then if (gApp%Model%IO%doTimerOut) call printClocks() - call cleanClocks() + doResetClocks = .true. endif if (gApp%Model%IO%doOutput(gApp%Model%t)) then @@ -274,6 +281,11 @@ program voltron_mpix call gApp%WriteRestart(gApp%Model%IO%nRes) endif + if (doResetClocks) then + call cleanClocks(gApp%Model%ts) + doResetClocks = .false. + endif + call Toc("IO") call Toc("Omega", .true.) end do diff --git a/src/drivers/voltronx.F90 b/src/drivers/voltronx.F90 index 8641d711..62e5357b 100644 --- a/src/drivers/voltronx.F90 +++ b/src/drivers/voltronx.F90 @@ -11,6 +11,7 @@ program voltronx type(voltApp_T) :: vApp real(rp) :: nextDT + logical :: doResetClocks = .false. call initClocks() @@ -37,10 +38,10 @@ program voltronx call consoleOutputV(vApp,vApp%gApp) !Timing info if (vApp%IO%doTimerOut) call printClocks() - call cleanClocks() + doResetClocks = .true. elseif (vApp%IO%doTimer(vApp%time)) then if (vApp%IO%doTimerOut) call printClocks() - call cleanClocks() + doResetClocks = .true. endif !Data output @@ -51,6 +52,11 @@ program voltronx if (vApp%IO%doRestart(vApp%time)) then call resOutputV(vApp,vApp%gApp) endif + !Reset clocks last + if (doResetClocks) then + call cleanClocks(vApp%ts) + doResetClocks = .false. + endif call Toc("IO", .true.) diff --git a/src/gamera/gamapp.F90 b/src/gamera/gamapp.F90 index 7aeb4d01..d5c82a84 100644 --- a/src/gamera/gamapp.F90 +++ b/src/gamera/gamapp.F90 @@ -82,11 +82,11 @@ module gamapp call Toc("DT") !Enforce BCs - call Tic("BCs") + call Tic("BCs", .true.) call EnforceBCs(gameraApp%Model,gameraApp%Grid,gameraApp%State) !Update Bxyz's call bFlux2Fld (gameraApp%Model,gameraApp%Grid,gameraApp%State%magFlux,gameraApp%State%Bxyz) - call Toc("BCs") + call Toc("BCs", .true.) end subroutine stepGamera diff --git a/src/gamera/gioH5.F90 b/src/gamera/gioH5.F90 index 9b77e211..9b5c92e9 100644 --- a/src/gamera/gioH5.F90 +++ b/src/gamera/gioH5.F90 @@ -258,7 +258,7 @@ module gioH5 type(State_T), intent(in) :: State character(len=*), intent(in) :: gStr - integer :: i,j,k,s + integer :: i,j,k,s,nClkSteps character(len=strLen) :: dID,VxID,VyID,VzID,PID integer iMin,iMax,jMin,jMax,kMin,kMax @@ -537,6 +537,17 @@ module gioH5 call AddOutVar(IOVars,"kzcsTOT",Model%kzcsTOT,uStr="kZCs",dStr="Total kZCs" ) !--------------------- + !Performance metrics + + nClkSteps = Model%ts - cleanClockStep + call AddOutVar(IOVars,"perf_stepTime",readClock(1)/nClkSteps) + call AddOutVar(IOVars,"perf_mathTime", readClock('Gamera')/nClkSteps) + call AddOutVar(IOVars,"perf_bcTime", readClock('BCs')/nClkSteps) + call AddOutVar(IOVars,"perf_haloTime", readClock('Halos')/nClkSteps) + call AddOutVar(IOVars,"perf_voltTime", readClock('VoltSync')/nClkSteps) + call AddOutVar(IOVars,"perf_ioTime", readClock('IO')/nClkSteps) + + !---------------------- !Call user routine if defined if (associated(Model%HackIO)) then diff --git a/src/gamera/mpi/gamapp_mpi.F90 b/src/gamera/mpi/gamapp_mpi.F90 index a4d7516d..00260c6b 100644 --- a/src/gamera/mpi/gamapp_mpi.F90 +++ b/src/gamera/mpi/gamapp_mpi.F90 @@ -578,9 +578,9 @@ module gamapp_mpi character(len=strLen) :: BCID !Enforce BCs - call Tic("BCs") + call Tic("BCs", .true.) call EnforceBCs(gamAppMpi%Model,gamAppMpi%Grid,State) - call Toc("BCs") + call Toc("BCs", .true.) !Track timing for all gamera ranks to finish physical BCs ! Only synchronize when timing @@ -591,10 +591,10 @@ module gamapp_mpi endif !Update ghost cells - call Tic("Halos") + call Tic("Halos", .true.) call HaloUpdate(gamAppMpi, State) call bFlux2Fld(gamAppMpi%Model, gamappMpi%Grid, State%magFlux, State%Bxyz) !Update Bxyz's - call Toc("Halos") + call Toc("Halos", .true.) !Track timing for all gamera ranks to finish halo comms ! Only synchronize when timing @@ -605,6 +605,7 @@ module gamapp_mpi endif ! Re-apply periodic BCs last + call Tic("BCs", .true.) do i=1,gamAppMpi%Grid%NumBC if(allocated(gamAppMpi%Grid%externalBCs(i)%p)) then SELECT type(bc=>gamAppMpi%Grid%externalBCs(i)%p) @@ -643,6 +644,7 @@ module gamapp_mpi endselect endif enddo + call Toc("BCs", .true.) !Track timing for all gamera ranks to finish periodic BCs ! Only synchronize when timing diff --git a/src/raiju/raijuAdvancer.F90 b/src/raiju/raijuAdvancer.F90 index 37dea347..a525c0e2 100644 --- a/src/raiju/raijuAdvancer.F90 +++ b/src/raiju/raijuAdvancer.F90 @@ -34,21 +34,21 @@ module raijuAdvancer State%dt = dtCpl - call Tic("Pre-Advance") + call Tic("Pre-Advance",.true.) call raijuPreAdvance(Model, Grid, State) - call Toc("Pre-Advance") + call Toc("Pre-Advance",.true.) State%isFirstCpl = .false. ! Step - call Tic("AdvanceState") + call Tic("AdvanceState",.true.) call AdvanceState(Model, Grid, State) - call Toc("AdvanceState") + call Toc("AdvanceState",.true.) ! etas back to moments - call Tic("Moments Eval") + call Tic("Moments Eval",.true.) call EvalMoments(Grid, State) call EvalMoments(Grid, State, doAvgO=.true.) - call Toc("Moments Eval") + call Toc("Moments Eval",.true.) end subroutine raijuAdvance diff --git a/src/raiju/raijuIO.F90 b/src/raiju/raijuIO.F90 index d4d82f99..c73eb1b1 100644 --- a/src/raiju/raijuIO.F90 +++ b/src/raiju/raijuIO.F90 @@ -154,7 +154,7 @@ module raijuIO logical, optional, intent(in) :: doGhostsO type(IOVAR_T), dimension(MAXIOVAR) :: IOVars - integer :: i,j,k,s + integer :: i,j,k,s, nClkSteps integer :: is, ie, js, je, ks, ke integer, dimension(4) :: outBnds2D logical :: doGhosts @@ -423,8 +423,14 @@ module raijuIO call AddOutVar(IOVars, "mapJacNorm", outTmp2D(is:ie,js:je), dStr="L_(2,1) norm of lat/lon => xyzMin Jacobian") endif - call WriteVars(IOVars,.true.,Model%raijuH5, gStr) + !Performance Metrics + nClkSteps = State%ts - cleanClockStep + call AddOutVar(IOVars, "perf_stepTime", readClock(1)/nClkSteps) + call AddOutVar(IOVars, "perf_preAdvance", readClock("Pre-Advance")/nClkSteps) + call AddOutVar(IOVars, "perf_advanceState", readClock("AdvanceState")/nClkSteps) + call AddOutVar(IOVars, "perf_moments", readClock("Moments Eval")/nClkSteps) + call WriteVars(IOVars,.true.,Model%raijuH5, gStr) ! Any extra groups to add if (Model%doLosses .and. Model%doOutput_3DLoss) then diff --git a/src/voltron/mpi/voltapp_mpi.F90 b/src/voltron/mpi/voltapp_mpi.F90 index 34807fcb..0e1da342 100644 --- a/src/voltron/mpi/voltapp_mpi.F90 +++ b/src/voltron/mpi/voltapp_mpi.F90 @@ -359,6 +359,7 @@ module voltapp_mpi call Tic("DeepUpdate") call DeepUpdate_mpi(vApp) call Toc("DeepUpdate") + vApp%ts = vApp%ts + 1 if(vApp%doSerialMHD) call vApp%gApp%StartUpdateMhdData(vApp) diff --git a/src/voltron/voltapp.F90 b/src/voltron/voltapp.F90 index b277f489..b8f8b18f 100644 --- a/src/voltron/voltapp.F90 +++ b/src/voltron/voltapp.F90 @@ -337,6 +337,7 @@ module voltapp call Tic("DeepUpdate") call DeepUpdate(vApp, vApp%gApp) call Toc("DeepUpdate") + vApp%ts = vApp%ts + 1 call vApp%gApp%StartUpdateMhdData(vApp) diff --git a/src/voltron/voltio.F90 b/src/voltron/voltio.F90 index 5fedd1e8..b4cebbaf 100644 --- a/src/voltron/voltio.F90 +++ b/src/voltron/voltio.F90 @@ -389,7 +389,7 @@ module voltio type(IOVAR_T), dimension(MAXVOLTIOVAR) :: IOVars real(rp) :: symh - integer :: is,ie,js,je + integer :: is,ie,js,je,nClkSteps real(rp) :: Csijk,Con(NVAR) real(rp) :: BSDst0,AvgBSDst,DPSDst,BSSMRs(4) integer, dimension(4) :: outSGVBnds_corner @@ -444,6 +444,18 @@ module voltio call AddOutVar(IOVars,"MJD" ,vApp%MJD) call AddOutVar(IOVars,"timestep",vApp%ts) + !Performance metrics + if(vApp%writePerf) then + nClkSteps = vApp%ts - cleanClockStep + call AddOutVar(IOVars,"perf_stepTime",readClock(1)/nClkSteps) + call AddOutVar(IOVars,"perf_gamTime", readClock('GameraSync')/nClkSteps) + call AddOutVar(IOVars,"perf_squishTime", (readClock('Squish')+readClock('VoltHelpers'))/nClkSteps) + call AddOutVar(IOVars,"perf_imagTime", readClock('InnerMag')/nClkSteps) + call AddOutVar(IOVars,"perf_mixTime", readClock('ReMIX')/nClkSteps) + call AddOutVar(IOVars,"perf_tubesTime", readClock('VoltTubes')/nClkSteps) + call AddOutVar(IOVars,"perf_ioTime", readClock('IO')/nClkSteps) + endif + ! voltState stuff call AddOutSGV(IOVars, "Potential_total", vApp%State%potential_total, & From bff7064a761ad0765f95654cbaf874f6248475ee Mon Sep 17 00:00:00 2001 From: Jeffrey Garretson Date: Thu, 21 Aug 2025 17:24:55 -0600 Subject: [PATCH 07/11] Fixing new test, removing rcm test executable once and for all --- testingScripts/unitTestReport.py | 6 +-- tests/cases_mpi/testCasesMpi.pf | 2 +- tests/rcm/testetautils.pf | 82 ----------------------------- tests/runNonCaseTests1-template.pbs | 7 --- tests/{rcm => voltron}/cmriD.xml | 0 5 files changed, 4 insertions(+), 93 deletions(-) delete mode 100644 tests/rcm/testetautils.pf rename tests/{rcm => voltron}/cmriD.xml (100%) diff --git a/testingScripts/unitTestReport.py b/testingScripts/unitTestReport.py index 36a086ce..2fdc61e4 100644 --- a/testingScripts/unitTestReport.py +++ b/testingScripts/unitTestReport.py @@ -138,7 +138,7 @@ def main(): # Compute the names of the job log files. job_file_0 = f"genTestData.o{job_ids[0]}" # 0 OKs job_file_1 = f"runCaseTests.o{job_ids[1]}" # 2 OKs - job_file_2 = f"runNonCaseTests1.o{job_ids[2]}" # 7 OKs + job_file_2 = f"runNonCaseTests1.o{job_ids[2]}" # 6 OKs job_file_3 = f"runNonCaseTests2.o{job_ids[3]}" # 1 OK if debug: print(f"job_file_0 = {job_file_0}") @@ -164,8 +164,8 @@ def main(): elif 'job killed' in line: jobKilled = True - # There should be exactly 10 OKs. - OK_COUNT_EXPECTED = 10 + # There should be exactly 9 OKs. + OK_COUNT_EXPECTED = 9 if verbose: print(f"Found {okCount} OKs, expected {OK_COUNT_EXPECTED}.") if okCount != OK_COUNT_EXPECTED: diff --git a/tests/cases_mpi/testCasesMpi.pf b/tests/cases_mpi/testCasesMpi.pf index 7f7d1b99..515bb16a 100644 --- a/tests/cases_mpi/testCasesMpi.pf +++ b/tests/cases_mpi/testCasesMpi.pf @@ -121,7 +121,7 @@ contains do j=1,2 do k=1,2 call AddInVar(IOVars,"P") - write(h5Str,'(A,I0,A)') 'blast3d_large8_0002_0002_0002_000',i-1,'_0000_0000.gam.h5' + write(h5Str,'(A,I0,A,I0,A,I0,A)') 'blast3d_large8_0002_0002_0002_000',i-1,'_000',j-1,'_000',k-1,'.gam.h5' call ReadVars(IOVars,.false.,h5Str,gStr) call IOArray3DFill(IOVars,"P",p8(1+(i-1)*ni2:i*ni2,1+(j-1)*nj2:j*nj2,1+(k-1)*nk2:k*nk2)) call ClearIO(IOVars) diff --git a/tests/rcm/testetautils.pf b/tests/rcm/testetautils.pf deleted file mode 100644 index cb1863bb..00000000 --- a/tests/rcm/testetautils.pf +++ /dev/null @@ -1,82 +0,0 @@ -module testetautils - use testHelper - use kdefs, only : TINY - use rcmdefs - use conversion_module, only : almmax,almmin - use RCM_mod_subs - use torcm_mod - use xml_input - use etautils - - implicit none - - contains - - @before - subroutine setup() - end subroutine setup - - @after - subroutine teardown() - end subroutine teardown - - !Helpers - subroutine initAlams() - real (rp) :: itimei,itimef - integer(iprec) :: nstep - type(XML_Input_T) :: xmlInp - INTEGER(iprec) :: ierr - - itimei = 0 - itimef = 60 - nstep = 0 - - xmlInp = New_XML_Input('cmriD.xml','Kaiju/Voltron',.true.) - - ! Get RCM to read rcmconfig.h5 and store in alamc - call setFactors(6.371E6_rprec) - call allocate_conversion_arrays (isize,jsize,kcsize) - call RCM(itimei, itimef, nstep, 0_iprec) - call RCM(itimei, itimef, nstep, 1_iprec) - call Read_alam (kcsize, alamc, ikflavc, fudgec, almdel, almmax, almmin, iesize, ierr) - end subroutine initAlams - - @test - subroutine testDPetaDP() - real (rp) :: etas(kcsize) - real (rp) :: vm - real (rp) :: Do_rc ,Po_rc - real (rp) :: Df_rc,Df_psph,Pf_rc - - real(rp) :: OKerr - character(len=strLen) :: checkMessage - - real(rp) :: D_err, P_err - - OKerr = 2e-2_rp - - ! 5 keV at L=10 - Do_rc = 0.5 *1E6! [1/cc -> 1/m^3] - Po_rc = 0.5 *1E-9! [nPa -> Pa] - vm = 2.262 - - call initAlams() - - call DP2eta(Do_rc,Po_rc,vm,etas,doRescaleO=.false.,doKapO=.false.) - call eta2DP(etas,vm,Df_rc,Df_psph,Pf_rc) - - D_err = (Do_rc - Df_rc)/Do_rc - P_err = (Po_rc - Pf_rc)/Po_rc - - write(*,*) 'D_err=',D_err*100,'%' - write(*,*) 'P_err=',P_err*100,'%' - - write (checkMessage,'(A,I0,A)') "D->eta->D' error > ",OKerr*100,"%" - @assertLessThanOrEqual(abs(D_err), OKerr, checkMessage) - - write (checkMessage,'(A,I0,A)') "D->eta->D' error > ",OKerr*100,"%" - @assertLessThanOrEqual(abs(P_err), OKerr, checkMessage) - - end subroutine testDPetaDP - -end module testetautils diff --git a/tests/runNonCaseTests1-template.pbs b/tests/runNonCaseTests1-template.pbs index 84dcb488..e3787eab 100644 --- a/tests/runNonCaseTests1-template.pbs +++ b/tests/runNonCaseTests1-template.pbs @@ -42,13 +42,6 @@ date echo 'REMIX tests complete.' echo | tail -n 3 ./mixTests.out -echo 'Running RCM tests.' -date -./rcmTests >& rcmTests.out -date -echo 'RCM tests complete.' -echo | tail -n 3 ./rcmTests.out - echo 'Running SHELLGRID tests.' date ./shgrTests >& shgrTests.out diff --git a/tests/rcm/cmriD.xml b/tests/voltron/cmriD.xml similarity index 100% rename from tests/rcm/cmriD.xml rename to tests/voltron/cmriD.xml From 7c93e4fc607192ae4d3b41012d298e6358b20869 Mon Sep 17 00:00:00 2001 From: cookieenick Date: Tue, 26 Aug 2025 11:13:45 -0600 Subject: [PATCH 08/11] GTR segmentation workflow updated --- scripts/makeitso/engage.py | 11 +++----- scripts/makeitso/makeitso.py | 55 +++++++++++++++++------------------- 2 files changed, 30 insertions(+), 36 deletions(-) diff --git a/scripts/makeitso/engage.py b/scripts/makeitso/engage.py index b64d3e80..1d8218d3 100755 --- a/scripts/makeitso/engage.py +++ b/scripts/makeitso/engage.py @@ -28,6 +28,7 @@ import json import os import sys import subprocess +import math # Import 3rd-party modules. import netCDF4 @@ -356,7 +357,7 @@ def prompt_user_for_run_options(args): for on in ["use_segments"]: od[on]["default"] = "Y" o[on] = makeitso.get_run_option(on, od[on], mode) - if o["use_segments"] == "Y": + if o["use_segments"].upper() == "Y": for on in ["segment_duration"]: o[on] = makeitso.get_run_option(on, od[on], mode) else: @@ -364,7 +365,7 @@ def prompt_user_for_run_options(args): # Compute the number of segments based on the simulation duration and # segment duration, add 1 if there is a remainder. - if o["use_segments"] == "Y": + if o["use_segments"].upper() == "Y": num_segments = simulation_duration/float(o["segment_duration"]) if num_segments > int(num_segments): num_segments += 1 @@ -564,11 +565,7 @@ def main(): segment_duration = float(engage_options["simulation"]["segment_duration"]) makeitso_options["voltron"]["time"]["tFin"] = int((t1-t0).total_seconds()) - num_segments = (t1-t0).total_seconds()/segment_duration - if num_segments > int(num_segments): - num_segments = int(num_segments) + 1 - else: - num_segments = int(num_segments) + num_segments = math.ceil((t1-t0).total_seconds()/segment_duration) makeitso_options["pbs"]["num_segments"] = str(num_segments) select2 = 1 + int(makeitso_options["pbs"]["num_helpers"]) makeitso_options["pbs"]["select2"] = str(select2) diff --git a/scripts/makeitso/makeitso.py b/scripts/makeitso/makeitso.py index 74d46e98..6075554d 100644 --- a/scripts/makeitso/makeitso.py +++ b/scripts/makeitso/makeitso.py @@ -33,6 +33,7 @@ import datetime import json import os import subprocess +import math # Import 3rd-party modules. import h5py @@ -889,27 +890,28 @@ def create_ini_files(options: dict, args: dict): # Set default value for padding to tFin for coupling. tfin_padding = 0.0 # Engage modifications to parameters. - # If TIEGCM coupling is specified, warmup segments are calculated - # based on gr_warm_up_time and segment duration. If the segment - # duration is not evenly divisible by gr_warm_up_time, the - # warmup segment duration is set to gr_warm_up_time/4. - # The number of warmup segments is set to gr_warm_up_time/ - # warmup_segment_duration. + # If TIEGCM coupling is specified, calculate warmup segments based + # on gr_warm_up_time and segment_duration. + # If gr_warm_up_time is an exact multiple of segment_duration, + # use segment_duration for each warmup segment. + # If gr_warm_up_time is less than segment_duration, use + # gr_warm_up_time as the duration for a single warmup segment. + # If gr_warm_up_time is greater than segment_duration but not an + # exact multiple, use segment_duration and round up the number of segments. if "coupling" in args: coupling = args["coupling"] gr_warm_up_time = float(coupling["gr_warm_up_time"]) segment_duration = float(options["simulation"]["segment_duration"]) simulation_duration = float(options["voltron"]["time"]["tFin"]) - i_last_warmup_ini = (gr_warm_up_time/segment_duration) - if i_last_warmup_ini == int(i_last_warmup_ini): + if gr_warm_up_time % segment_duration == 0: warmup_segment_duration = segment_duration - else: - warmup_segment_duration = gr_warm_up_time/4 - if warmup_segment_duration != int(warmup_segment_duration): - print("Error: gr_warm_up_time is not evenly divisible by 4.") - raise ValueError("Invalid gr_warm_up_time value.") - i_last_warmup_ini = (gr_warm_up_time/warmup_segment_duration) - i_last_warmup_ini = int(i_last_warmup_ini) + i_last_warmup_ini = int(gr_warm_up_time/warmup_segment_duration) + elif gr_warm_up_time < segment_duration: + warmup_segment_duration = gr_warm_up_time + i_last_warmup_ini = int(gr_warm_up_time/warmup_segment_duration) + elif gr_warm_up_time > segment_duration: + warmup_segment_duration = segment_duration + i_last_warmup_ini = int(math.ceil(gr_warm_up_time/segment_duration)) # Add padding to tFin for coupling. if coupling["tfin_delta"] == "T": tfin_coupling_padding = float(options["voltron"]["coupling"]["dtCouple"]) - 1 @@ -986,11 +988,7 @@ def create_ini_files(options: dict, args: dict): # Create an .ini file for each simulation segment. Files for each # segment will be numbered starting with 1. if "coupling" in args: - num_segments = (simulation_duration - num_warmup_segments*warmup_segment_duration)/segment_duration - if num_segments > int(num_segments): - num_segments = int(num_segments) + 1 - else: - num_segments = int(num_segments) + num_segments = math.ceil((simulation_duration - num_warmup_segments*warmup_segment_duration)/segment_duration) else: num_segments = int(options["pbs"]["num_segments"]) for job in range(1, num_segments + 1): @@ -1218,16 +1216,15 @@ def create_pbs_scripts(xml_files: list, options: dict, args: dict): gr_warm_up_time = float(coupling["gr_warm_up_time"]) segment_duration = float(options["simulation"]["segment_duration"]) simulation_duration = float(options["voltron"]["time"]["tFin"]) - i_last_warmup_ini = (gr_warm_up_time/segment_duration) - if i_last_warmup_ini == int(i_last_warmup_ini): + if gr_warm_up_time % segment_duration == 0: warmup_segment_duration = segment_duration - else: - warmup_segment_duration = gr_warm_up_time/4 - if warmup_segment_duration != int(warmup_segment_duration): - print("Error: gr_warm_up_time is not evenly divisible by 4.") - raise ValueError("Invalid gr_warm_up_time value.") - i_last_warmup_ini = (gr_warm_up_time/warmup_segment_duration) - i_last_warmup_ini = int(i_last_warmup_ini) + i_last_warmup_ini = int(gr_warm_up_time/warmup_segment_duration) + elif gr_warm_up_time < segment_duration: + warmup_segment_duration = gr_warm_up_time + i_last_warmup_ini = int(gr_warm_up_time/warmup_segment_duration) + elif gr_warm_up_time > segment_duration: + warmup_segment_duration = segment_duration + i_last_warmup_ini = int(math.ceil(gr_warm_up_time/segment_duration)) num_warmup_segments = i_last_warmup_ini #i_last_warmup_pbs_script = int(gr_warm_up_time/segment_duration) spinup_pbs_scripts.append(pbs_scripts[0]) # Spinup script is first From c8a95cf525a7ac6e9be82fd2608e1e2cbe305351 Mon Sep 17 00:00:00 2001 From: Jeffrey Garretson Date: Tue, 2 Sep 2025 14:09:36 -0600 Subject: [PATCH 09/11] Tweaks to support tests --- .gitignore | 4 ++++ testingScripts/weeklyDashGo.xml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e79087e0..d33369ce 100644 --- a/.gitignore +++ b/.gitignore @@ -8,11 +8,15 @@ external/FARGPARSE-*/ external/GFTL-*/ external/GFTL_SHARED-*/ external/PFUNIT-*/ + # skip F90 files in the tests folder, except in specific subfolders tests/*/*.F90 !tests/helperCode/*.F90 !tests/helperCode_mpi/*.F90 +# any local automated tests that users have run +test_runs/ + # Pre-compile generated files src/base/git_info.F90 diff --git a/testingScripts/weeklyDashGo.xml b/testingScripts/weeklyDashGo.xml index 87b18f9d..78b43d8e 100644 --- a/testingScripts/weeklyDashGo.xml +++ b/testingScripts/weeklyDashGo.xml @@ -8,7 +8,7 @@ - + From aa43839b8cc95bc6e23dd237a0cd46d41d5beeef Mon Sep 17 00:00:00 2001 From: Jeffrey Garretson Date: Tue, 2 Sep 2025 21:28:38 -0600 Subject: [PATCH 10/11] Tweaks to h5 performance output --- src/base/clocks.F90 | 50 ++++++++++++++++++++++++++++----- src/base/types/volttypes.F90 | 1 - src/drivers/voltron_mpix.F90 | 4 +-- src/drivers/voltronx.F90 | 2 +- src/gamera/gamapp.F90 | 3 ++ src/gamera/gioH5.F90 | 15 +++++----- src/gamera/mpi/gamapp_mpi.F90 | 2 ++ src/raiju/raijuIO.F90 | 10 +++---- src/voltron/mpi/voltapp_mpi.F90 | 11 ++------ src/voltron/voltapp.F90 | 4 +-- src/voltron/voltio.F90 | 20 ++++++------- 11 files changed, 77 insertions(+), 45 deletions(-) diff --git a/src/base/clocks.F90 b/src/base/clocks.F90 index cafdb0e3..94b5385c 100644 --- a/src/base/clocks.F90 +++ b/src/base/clocks.F90 @@ -8,7 +8,6 @@ module clocks !Global clock parameters integer, parameter :: maxClocks = 80 integer :: clockRate=0,clockMax=0 - integer :: cleanClockStep=0 !Clock output (min/max depth) integer :: clkMinD = 1, clkMaxD = 5 @@ -39,6 +38,7 @@ module clocks !Depth/parent, ie array entry of parent of this clock integer :: level=-1,parent=-1 integer :: iTic=0,iToc=0 !Integer ticks + integer :: nCalls=0 !Number of tocs, or finished timer loops since cleaning real(rp) :: tElap=0.0 !Elapsed time end type Clock_T @@ -51,6 +51,11 @@ module clocks interface readClock module procedure readClock_str, readClock_int end interface + + !interface for reading number of calls to a clock + interface readNCalls + module procedure readNCalls_str, readNCalls_int + end interface contains @@ -172,19 +177,17 @@ module clocks wclk = real(kClocks(iblk)%iToc-kClocks(iblk)%iTic)/real(clockRate) kClocks(iblk)%tElap = kClocks(iblk)%tElap + wclk + kClocks(iblk)%nCalls = kClocks(iblk)%nCalls + 1 + end subroutine Toc !Reset clocks - subroutine cleanClocks(cleanStep) - integer, optional, intent(in) :: cleanStep + subroutine cleanClocks() integer :: n - ! optional, allow the user to record what model step the clocks were cleaned at - ! this can help the user track per-step timing info - if(present(cleanStep)) cleanClockStep = cleanStep - do n=1,nclk kClocks(n)%tElap = 0 + kClocks(n)%nCalls = 0 ! if the clock is active, reset the tic to right now if(kClocks(n)%isOn) call Tic(kClocks(n)%cID, .true.) enddo @@ -229,6 +232,39 @@ module clocks endif end function readClock_int + function readNCalls_str(cID) result(nc) + character(len=*), intent(in) :: cID + + integer :: n,iblk + integer :: nc + + iblk = 0 + !Find timer + do n=1,nclk + if (toUpper(kClocks(n)%cID) == toUpper(cID)) then + !Found it, save ID + iblk = n + endif + enddo + + nc = readNCalls_int(iblk) + + end function readNCalls_str + + function readNCalls_int(iblk) result(nc) + integer, intent(in) :: iblk + + integer :: tmpToc + integer :: nc + + if (iblk == 0) then + nc = 0 + else + nc = kClocks(iblk)%nCalls + endif + + end function readNCalls_int + !Output clock data subroutine printClocks() integer :: n,l diff --git a/src/base/types/volttypes.F90 b/src/base/types/volttypes.F90 index f0dcfda3..2acfdd26 100644 --- a/src/base/types/volttypes.F90 +++ b/src/base/types/volttypes.F90 @@ -354,7 +354,6 @@ module volttypes type (IOClock_T) :: IO logical :: isLoud = .true. !Console output logical :: writeFiles = .true. !File output - logical :: writePerf = .true. ! Write performance data to files !Apps type(mixApp_T) :: remixApp diff --git a/src/drivers/voltron_mpix.F90 b/src/drivers/voltron_mpix.F90 index aa9fbddf..08456ad3 100644 --- a/src/drivers/voltron_mpix.F90 +++ b/src/drivers/voltron_mpix.F90 @@ -227,7 +227,7 @@ program voltron_mpix !Reset clocks last so data is available for all output if (doResetClocks) then - call cleanClocks(vApp%ts) + call cleanClocks() doResetClocks = .false. endif @@ -282,7 +282,7 @@ program voltron_mpix endif if (doResetClocks) then - call cleanClocks(gApp%Model%ts) + call cleanClocks() doResetClocks = .false. endif diff --git a/src/drivers/voltronx.F90 b/src/drivers/voltronx.F90 index 62e5357b..6ba56d2b 100644 --- a/src/drivers/voltronx.F90 +++ b/src/drivers/voltronx.F90 @@ -54,7 +54,7 @@ program voltronx endif !Reset clocks last if (doResetClocks) then - call cleanClocks(vApp%ts) + call cleanClocks() doResetClocks = .false. endif diff --git a/src/gamera/gamapp.F90 b/src/gamera/gamapp.F90 index d5c82a84..639273e0 100644 --- a/src/gamera/gamapp.F90 +++ b/src/gamera/gamapp.F90 @@ -73,6 +73,7 @@ module gamapp subroutine stepGamera(gameraApp) class(gamApp_T), intent(inout) :: gameraApp + call Tic("Advance", .true.) !update the state variables to the next timestep call UpdateStateData(gameraApp) @@ -87,6 +88,8 @@ module gamapp !Update Bxyz's call bFlux2Fld (gameraApp%Model,gameraApp%Grid,gameraApp%State%magFlux,gameraApp%State%Bxyz) call Toc("BCs", .true.) + + call Toc("Advance", .true.) end subroutine stepGamera diff --git a/src/gamera/gioH5.F90 b/src/gamera/gioH5.F90 index b3168e28..6d396b5e 100644 --- a/src/gamera/gioH5.F90 +++ b/src/gamera/gioH5.F90 @@ -541,13 +541,14 @@ module gioH5 !--------------------- !Performance metrics - nClkSteps = Model%ts - cleanClockStep - call AddOutVar(IOVars,"perf_stepTime",readClock(1)/nClkSteps) - call AddOutVar(IOVars,"perf_mathTime", readClock('Gamera')/nClkSteps) - call AddOutVar(IOVars,"perf_bcTime", readClock('BCs')/nClkSteps) - call AddOutVar(IOVars,"perf_haloTime", readClock('Halos')/nClkSteps) - call AddOutVar(IOVars,"perf_voltTime", readClock('VoltSync')/nClkSteps) - call AddOutVar(IOVars,"perf_ioTime", readClock('IO')/nClkSteps) + nClkSteps = readNCalls('Advance') + call AddOutVar(IOVars,"_perf_stepTime",readClock(1)/nClkSteps) + call AddOutVar(IOVars,"_perf_mathTime", readClock('Gamera')/nClkSteps) + call AddOutVar(IOVars,"_perf_bcTime", readClock('BCs')/nClkSteps) + call AddOutVar(IOVars,"_perf_haloTime", readClock('Halos')/nClkSteps) + call AddOutVar(IOVars,"_perf_voltTime", readClock('VoltSync')/nClkSteps) + call AddOutVar(IOVars,"_perf_ioTime", readClock('IO')/nClkSteps) + call AddOutVar(IOVars,"_perf_advanceTime", readClock('Advance')/nClkSteps) !---------------------- diff --git a/src/gamera/mpi/gamapp_mpi.F90 b/src/gamera/mpi/gamapp_mpi.F90 index 3ae08824..31c2c607 100644 --- a/src/gamera/mpi/gamapp_mpi.F90 +++ b/src/gamera/mpi/gamapp_mpi.F90 @@ -668,6 +668,7 @@ module gamapp_mpi integer :: ierr,i real(rp) :: tmp + call Tic("Advance", .true.) !update the state variables to the next timestep call UpdateStateData(gamAppMpi) @@ -686,6 +687,7 @@ module gamapp_mpi !Update BCs MPI style call updateMpiBCs(gamAppMpi, gamAppmpi%State) + call Toc("Advance", .true.) end subroutine stepGamera_mpi diff --git a/src/raiju/raijuIO.F90 b/src/raiju/raijuIO.F90 index 05a01184..889f8ea3 100644 --- a/src/raiju/raijuIO.F90 +++ b/src/raiju/raijuIO.F90 @@ -425,11 +425,11 @@ module raijuIO endif !Performance Metrics - nClkSteps = State%ts - cleanClockStep - call AddOutVar(IOVars, "perf_stepTime", readClock(1)/nClkSteps) - call AddOutVar(IOVars, "perf_preAdvance", readClock("Pre-Advance")/nClkSteps) - call AddOutVar(IOVars, "perf_advanceState", readClock("AdvanceState")/nClkSteps) - call AddOutVar(IOVars, "perf_moments", readClock("Moments Eval")/nClkSteps) + nClkSteps = readNCalls('DeepUpdate') + call AddOutVar(IOVars, "_perf_stepTime", readClock(1)/nClkSteps) + call AddOutVar(IOVars, "_perf_preAdvance", readClock("Pre-Advance")/nClkSteps) + call AddOutVar(IOVars, "_perf_advanceState", readClock("AdvanceState")/nClkSteps) + call AddOutVar(IOVars, "_perf_moments", readClock("Moments Eval")/nClkSteps) call WriteVars(IOVars,.true.,Model%raijuH5, gStr) diff --git a/src/voltron/mpi/voltapp_mpi.F90 b/src/voltron/mpi/voltapp_mpi.F90 index e51d8c52..dc32c8c6 100644 --- a/src/voltron/mpi/voltapp_mpi.F90 +++ b/src/voltron/mpi/voltapp_mpi.F90 @@ -356,9 +356,9 @@ module voltapp_mpi if(.not. vApp%doSerialMHD) call vApp%gApp%StartUpdateMhdData(vApp) - call Tic("DeepUpdate") + call Tic("DeepUpdate",.true.) call DeepUpdate_mpi(vApp) - call Toc("DeepUpdate") + call Toc("DeepUpdate",.true.) vApp%ts = vApp%ts + 1 if(vApp%doSerialMHD) call vApp%gApp%StartUpdateMhdData(vApp) @@ -405,8 +405,6 @@ module voltapp_mpi ! only do imag after spinup if(vApp%doDeep .and. vApp%time >= 0) then - call Tic("DeepUpdate", .true.) - if(vApp%useHelpers) call vhReqStep(vApp) ! instead of PreDeep, use Tube Helpers and replicate other calls @@ -443,7 +441,6 @@ module voltapp_mpi call DoImag(vApp) vApp%deepProcessingInProgress = .true. - call Toc("DeepUpdate", .true.) elseif(vApp%doDeep) then vApp%gApp%Grid%Gas0 = 0 !Load TM03 into Gas0 for ingestion during spinup @@ -460,7 +457,6 @@ module voltapp_mpi ! only do imag after spinup with deep enabled if(vApp%doDeep .and. vApp%time >= 0) then - call Tic("DeepUpdate", .true.) do while(SquishBlocksRemain(vApp)) call Tic("Squish",.true.) @@ -478,7 +474,6 @@ module voltapp_mpi call SquishEnd(vApp) call PostDeep(vApp, vApp%gApp) - call Toc("DeepUpdate", .true.) endif end subroutine endDeep @@ -497,11 +492,9 @@ module voltapp_mpi if(.not. vApp%deepProcessingInProgress) return if(SquishBlocksRemain(vApp)) then - call Tic("DeepUpdate") call Tic("Squish",.true.) call DoSquishBlock(vApp) call Toc("Squish",.true.) - call Toc("DeepUpdate") endif if(.not. SquishBlocksRemain(vApp)) then diff --git a/src/voltron/voltapp.F90 b/src/voltron/voltapp.F90 index b2471a9e..b87e3ea0 100644 --- a/src/voltron/voltapp.F90 +++ b/src/voltron/voltapp.F90 @@ -334,9 +334,9 @@ module voltapp ! update the next predicted coupling interval vApp%DeepT = vApp%DeepT + vApp%DeepDT - call Tic("DeepUpdate") + call Tic("DeepUpdate",.true.) call DeepUpdate(vApp, vApp%gApp) - call Toc("DeepUpdate") + call Toc("DeepUpdate",.true.) vApp%ts = vApp%ts + 1 call vApp%gApp%StartUpdateMhdData(vApp) diff --git a/src/voltron/voltio.F90 b/src/voltron/voltio.F90 index b4cebbaf..61264476 100644 --- a/src/voltron/voltio.F90 +++ b/src/voltron/voltio.F90 @@ -445,17 +445,15 @@ module voltio call AddOutVar(IOVars,"timestep",vApp%ts) !Performance metrics - if(vApp%writePerf) then - nClkSteps = vApp%ts - cleanClockStep - call AddOutVar(IOVars,"perf_stepTime",readClock(1)/nClkSteps) - call AddOutVar(IOVars,"perf_gamTime", readClock('GameraSync')/nClkSteps) - call AddOutVar(IOVars,"perf_squishTime", (readClock('Squish')+readClock('VoltHelpers'))/nClkSteps) - call AddOutVar(IOVars,"perf_imagTime", readClock('InnerMag')/nClkSteps) - call AddOutVar(IOVars,"perf_mixTime", readClock('ReMIX')/nClkSteps) - call AddOutVar(IOVars,"perf_tubesTime", readClock('VoltTubes')/nClkSteps) - call AddOutVar(IOVars,"perf_ioTime", readClock('IO')/nClkSteps) - endif - + nClkSteps = readNCalls('DeepUpdate') + call AddOutVar(IOVars,"_perf_stepTime",readClock(1)/nClkSteps) + call AddOutVar(IOVars,"_perf_deepUpdateTime",readClock(1)/nClkSteps) + call AddOutVar(IOVars,"_perf_gamTime", readClock('GameraSync')/nClkSteps) + call AddOutVar(IOVars,"_perf_squishTime", (readClock('Squish')+readClock('VoltHelpers'))/nClkSteps) + call AddOutVar(IOVars,"_perf_imagTime", readClock('InnerMag')/nClkSteps) + call AddOutVar(IOVars,"_perf_mixTime", readClock('ReMIX')/nClkSteps) + call AddOutVar(IOVars,"_perf_tubesTime", readClock('VoltTubes')/nClkSteps) + call AddOutVar(IOVars,"_perf_ioTime", readClock('IO')/nClkSteps) ! voltState stuff call AddOutSGV(IOVars, "Potential_total", vApp%State%potential_total, & From 8a42ad3c7828568ccefe9632b244b645b8fc4ae2 Mon Sep 17 00:00:00 2001 From: Jeffrey Garretson Date: Thu, 4 Sep 2025 11:33:48 -0600 Subject: [PATCH 11/11] Making local tests easier to run --- testingScripts/runLocalTests.py | 69 ++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 23 deletions(-) diff --git a/testingScripts/runLocalTests.py b/testingScripts/runLocalTests.py index 5da01b6e..e071f6f8 100755 --- a/testingScripts/runLocalTests.py +++ b/testingScripts/runLocalTests.py @@ -20,11 +20,11 @@ def create_command_line_parser(): parser = argparse.ArgumentParser(description="Script to help setup automated tests within a kaiju repo") parser.add_argument( - "-A", required=True, + "-A", default="", help="Charge code to use when running tests." ) parser.add_argument( - "-ce", required=True, + "-ce", default="", help="Conda environment name to load with conda module" ) parser.add_argument( @@ -80,23 +80,58 @@ def main(): # Parse the command-line arguments. args = parser.parse_args() + + # Adjust test options + if args.all: + args.unitTests = True + args.weeklyDash = True + args.compTests = True + args.compTestsFull = True + args.buildTests = True + args.icTests = True + args.intelChecks = True + args.reproTests = True + + if args.compTestsFull: + args.compTests = False + if not (args.unitTests or args.weeklyDash or args.compTests or args.compTestsFull or + args.buildTests or args.icTests or args.intelChecks or args.reproTests): + parser.print_help() + exit() + # find repo home directory called_from = os.path.dirname(os.path.abspath(__file__)) os.chdir(called_from) os.chdir('..') homeDir = os.getcwd() - + # Check for necessary environment variables - if 'KAIJUHOME' not in os.environ: - print("The setupEnvironment.sh script must be sourced for the repo this script resides in before calling it.") + if len(args.ce) == 0 and 'CONDA_DEFAULT_ENV' not in os.environ: + print("A conda environment name was not supplied, and a currently loaded conda environment could not be determined.") + print("Please either supply the name of a conda environment with the '-ce ' option,") + print(" or load an entironment before running this script, and it should be automatically found.") exit() + elif len(args.ce) == 0: + args.ce = os.environ['CONDA_DEFAULT_ENV'] + print(f"Automatically setting conda environment to {args.ce}") + if len(args.A) == 0 and (args.unitTests or args.weeklyDash or + args.compTests or args.compTestsFull or args.intelChecks or args.reproTests): + print("A charge code with not supplied, but the requested tests require one.") + print("Please supply a charge code with the -A # option.") + exit() + if 'KAIJUHOME' not in os.environ: + os.environ['KAIJUHOME'] = homeDir + print(f"Running tests out of local git repository: {homeDir}") if pathlib.Path(homeDir).resolve() != pathlib.Path(os.environ['KAIJUHOME']).resolve(): print("The setupEnvironment.sh script must be sourced for the repo this script resides in before calling it.") exit() - if 'KAIPYHOME' not in os.environ: - print("The setupEnvironment.sh script for ANY kaipy repo must be sourced before calling this.") + if 'KAIPYHOME' not in os.environ and (args.weeklyDash or args.compTests or args.compTestsFull): + print("The 'KAIPYHOME' environment variable was not set, but the requested tests require it.") + print("The setupEnvironment.sh script for ANY kaipy repo must be sourced before running these tests.") exit() + elif 'KAIPYHOME' not in os.environ: + os.environ['KAIPYHOME'] = "" # Set environment variables os.environ['MAGE_TEST_ROOT'] = homeDir @@ -124,22 +159,10 @@ def main(): print(f"Running tests on branch {gitBranch}") - print(f"Using charge code {args.A} with priority {args.p}") - print(f"Running in folder {test_set_dir}") - - # Adjust test options - if args.all: - args.unitTests = True - args.weeklyDash = True - args.compTests = True - args.compTestsFull = True - args.buildTests = True - args.icTests = True - args.intelChecks = True - args.reproTests = True - - if args.compTestsFull: - args.compTests = False + if len(args.A) > 0: + print(f"Using charge code {args.A} with priority {args.p}") + print(f"Running in folder test_runs/{test_set_dir}") + print("") # Run Tests if args.unitTests: