mirror of
https://github.com/JHUAPL/kaiju.git
synced 2026-01-08 22:58:05 -05:00
168 lines
6.8 KiB
Bash
168 lines
6.8 KiB
Bash
#!/bin/bash
|
|
|
|
# This PBS script illustrates how to run the helio_serial quickstart case
|
|
# using the serial version of the kaiju software.
|
|
|
|
# The lines starting with "#PBS" are directives for the PBS job control
|
|
# system, used on pleiades and derecho.
|
|
|
|
# On pleiades, this job should run in roughly 110 minutes of wall-clock time.
|
|
# On derecho, this job should run in roughly 137 minutes of wall-clock time.
|
|
|
|
# Example usage
|
|
# qsub helio_serial.pbs
|
|
|
|
# IMPORTANT: You *must* do the following in the code below:
|
|
# 1. Uncomment the #PBS lines for your system, comment out the #PBS lines
|
|
# specific to other systems. Keep the common #PBS lines.
|
|
# 2. (derecho only) Set the #PBS -A directive to your own account.
|
|
# 3. Uncomment the modules lines for your system, comment out the others.
|
|
# 4. Set kaiju_install_dir to your local kaiju installation.
|
|
# 5. Set PATH to include your bin subdirectory of the build subdirectory of
|
|
# your local kaiju installation.
|
|
# 6. If using the Intel compiler, uncomment the setting for KMP_STACK_SIZE and
|
|
# comment out setting OMP_STACK_SIZE. If not using the Intel compiler, do
|
|
# the opposite - uncomment OMP_STACK_SIZE, comment out KMP_STACK_SIZE.
|
|
|
|
#------------------------------------------------------------------------------
|
|
# START OF PBS DIRECTIVES
|
|
#------------------------------------------------------------------------------
|
|
|
|
# PBS directives for all systems
|
|
|
|
# Provide a useful name for this PBS job. This name will appear in the "qstat"
|
|
# output for this job.
|
|
#PBS -N helio_serial
|
|
|
|
# Combine job script std(o)ut and std(e)rr into stdout. This combined
|
|
# output will be saved in the file helio_serial.oJOB_NUM, where JOB_NUM is the
|
|
# PBS job number assigned when the job is submitted. Note that the output from
|
|
# the kaiju software itself is captured in a separate file (see below).
|
|
#PBS -j oe
|
|
|
|
# Request compute time on the allocated compute node(s). The format is
|
|
# hh:mm:ss, where hh is hours, mm is minutes, and ss is seconds. Set this
|
|
# value high enough to ensure that your job runs to completion, but low enough
|
|
# to stay within the time limits of the queue you are submitting to.
|
|
#PBS -l walltime=02:00:00
|
|
|
|
# Send an email to the user when the job is (a)borted, (b)egins, and/or (e)nds.
|
|
# An alternate email address may be specified with the #PBS -M directive.
|
|
#PBS -m abe
|
|
|
|
#-----------------------------------------------------------------------------
|
|
|
|
# System-specific PBS directives
|
|
|
|
# In the lines below, uncomment the set of PBS directives appropriate for your
|
|
# system, and comment out other sets. The default is pleiades.
|
|
|
|
#-----------------------------------------------------------------------------
|
|
|
|
# For the pleiades system at NASA Ames HECC:
|
|
|
|
# Submit the job to the PBS queue called "normal".
|
|
#PBS -q normal
|
|
|
|
# This is the line where you request specific resources from the PBS system.
|
|
# select=1 -> Request 1 compute node.
|
|
# ncpus=28 -> Each compute node must have at least 28 cores. This requirement
|
|
# implies the use of the 2-socket, 14 cores/socket Broadwell nodes.
|
|
# ompthreads=28 -> Each node will run 28 OMP threads. Note that this will
|
|
# also cause the OMP_THREADS environment variable to be set to 28.
|
|
# model=bro -> Each compute node must contain Broadwell chips. Specifying
|
|
# "model" is a HECC-specific PBS requirement.
|
|
#PBS -l select=1:ncpus=28:ompthreads=28:model=bro
|
|
|
|
#-----------------------------------------------------------------------------
|
|
|
|
# For the derecho system at UCAR:
|
|
|
|
# You *must* set the -A directive to your account number.
|
|
##PBS -A UJHB0019
|
|
##PBS -q main
|
|
##PBS -l select=1:ncpus=128:ompthreads=128
|
|
|
|
#------------------------------------------------------------------------------
|
|
# END OF PBS DIRECTIVES
|
|
#------------------------------------------------------------------------------
|
|
|
|
echo "Job $PBS_JOBID started at `date` on `hostname`."
|
|
|
|
# Specify the ID string for the run. This can be set to any desired string.
|
|
# PBS_JOBNAME is used here as an example, as it is set by the #PBS -N
|
|
# directive near the top of this file.
|
|
runid=$PBS_JOBNAME
|
|
|
|
#------------------------------------------------------------------------------
|
|
# START OF MODULE DIRECTIVES
|
|
#------------------------------------------------------------------------------
|
|
|
|
# Load the required modules for serial kaiju.
|
|
|
|
# NOTE: This set of modules assumes your kaiju installation was built
|
|
# using this same list of modules. If you used different modules at
|
|
# build time (for example, if you used a GNU compiler), update this
|
|
# list to use the modules from your build-time environment.
|
|
|
|
# Comment out the module lines for all systems except the one you are using.
|
|
module purge
|
|
|
|
# For pleiades:
|
|
module load nas
|
|
module load pkgsrc/2022Q1-rome # For git-lfs and cmake
|
|
module load comp-intel/2020.4.304 # Latest version
|
|
module load hdf5/1.8.18_serial
|
|
|
|
# For derecho:
|
|
# module load ncarenv/23.06
|
|
# module load craype/2.7.20
|
|
# module load intel/2023.0.0
|
|
# module load ncarcompilers/1.0.0 # Must come after intel/2023.0.0
|
|
# module load hdf5/1.12.2
|
|
# module load cmake/3.26.3
|
|
# module load geos/3.9.1 # Must come after intel/2023.0.0
|
|
|
|
echo "The following modules are loaded:"
|
|
module list
|
|
|
|
#------------------------------------------------------------------------------
|
|
# END OF MODULE DIRECTIVES
|
|
#------------------------------------------------------------------------------
|
|
|
|
# Define the kaiju installation location.
|
|
# NOTE: You MUST set this variable to the path to your kaiju directory, which
|
|
# is the top-level directory created when you cloned the kaiju repository.
|
|
kaiju_install_dir=$HOME/kaiju
|
|
|
|
# This script sets KAIJUHOME and other environment variables.
|
|
source $kaiju_install_dir/scripts/setupEnvironment.sh
|
|
|
|
# Add the kaiju binary directory to the command path.
|
|
# NOTE: You *must* set this variable to the path to the bin subdirectory of
|
|
# your kaiju build directory. The setting below assumes that the serial version
|
|
# of kaiju was built in the build_serial subdirectory of the kaiju home
|
|
# directory (which is typically the same as kaiju_install_dir).
|
|
export PATH=$KAIJUHOME/build_serial/bin:$PATH
|
|
|
|
# Set the OMP stack size to prevent a crash.
|
|
# If this setting is ignored, the model may cause the kaiju code to crash
|
|
# with a segmentation fault and core dump. The value of "128M" was chosen
|
|
# ~arbitrarily; experimentation may allow a smaller value to be used.
|
|
# NOTE: OMP_STACKSIZE is the standard OpenMP environment variable for setting
|
|
# the OMP stack size. If using the Intel compiler, comment out the
|
|
# OMP_STACKSIZE line and uncomment the KMP_STACKSIZE line. KMP_STACKSIZE is the
|
|
# Intel-specific environment variable for setting the OMP stack size.
|
|
# export OMP_STACKSIZE=128M
|
|
export KMP_STACKSIZE=128M
|
|
|
|
echo "The active environment variables are:"
|
|
printenv
|
|
|
|
# Run the model. Direct output from the program is saved in a text file.
|
|
EXE=gamhelio.x
|
|
echo "Running $EXE on model $runid."
|
|
$EXE $runid.xml >& ${EXE}.${runid}.out
|
|
|
|
echo "Job $PBS_JOBID ended at `date` on `hostname`."
|