#!/bin/bash

# This PBS script illustrates how to run the loop2d quickstart case using the
# serial version of the kaiju software.

# The lines starting with "#PBS" are directives for the PBS job control
# system, used on pleiades and derecho.

# On pleiades, this job should run in roughly 1 minute of wall-clock time.
# On derecho, this job should run in roughly 1 minute of wall-clock time.

# Example usage
# qsub loop2d.pbs

# IMPORTANT: You *must* do the following:
# 1. Uncomment the #PBS lines for your system, comment out the #PBS lines
#    specific to other systems. Keep the common #PBS lines.
# 2. (derecho only) Set the #PBS -A directive to your own account.
# 3. Uncomment the modules lines for your system, comment out the others.
# 4. Set kaiju_install_dir to your local kaiju installation.
# 5. Set PATH to include your bin subdirectory of the build subdirectory of
#    your local kaiju installation.
# 6. If using the Intel compiler, uncomment the setting for KMP_STACK_SIZE and
#    comment out setting OMP_STACK_SIZE. If not using the Intel compiler, do
#    the opposite - uncomment OMP_STACK_SIZE, comment out KMP_STACK_SIZE.

#------------------------------------------------------------------------------
# START OF PBS DIRECTIVES
#------------------------------------------------------------------------------

# PBS directives for all systems

# Provide a useful name for this PBS job. This name will appear in the "qstat"
# output for this job.
#PBS -N loop2d

# Combine job script std(o)ut and std(e)rr into stdout. This combined
# output will be saved in the file loop2d.oJOB_NUM, where JOB_NUM is the PBS
# job number assigned when the job is submitted. Note that the output from the
# kaiju software itself is captured in a separate file (see below).
#PBS -j oe

# Request compute time on the allocated compute node(s). The format is
# hh:mm:ss, where hh is hours, mm is minutes, and ss is seconds. Set this
# value high enough to ensure that your job runs to completion, but low enough
# to stay within the time limits of the queue you are submitting to.
#PBS -l walltime=00:05:00

# Send an email to the user when the job is (a)borted, (b)egins, and/or (e)nds.
# An alternate email address may be specified with the #PBS -M directive.
#PBS -m abe

#-----------------------------------------------------------------------------

# System-specific PBS directives

# In the lines below, uncomment the set of PBS directives appropriate for your
# system, and comment out other sets. The default is pleiades.

#-----------------------------------------------------------------------------

# For the pleiades system at NASA Ames HECC:

# Submit the job to the PBS queue called "normal".
#PBS -q normal

# This is the line where you request specific resources from the PBS system.
# select=1 -> Request 1 compute node.
# ncpus=28 -> Each compute node must have at least 28 cores. This requirement
# implies the use of the 2-socket, 14 cores/socket Broadwell nodes.
# ompthreads=28 -> Each node will run 28 OMP threads. Note that this will
# also cause the OMP_THREADS environment variable to be set to 28.
# model=bro -> Each compute node must contain Broadwell chips. Specifying
# "model" is a HECC-specific PBS requirement.
#PBS -l select=1:ncpus=28:ompthreads=28:model=bro

#-----------------------------------------------------------------------------

# For the derecho system at UCAR:

# You *must* set the -A directive to your account number.
##PBS -A UJHB0019
##PBS -q main
##PBS -l select=1:ncpus=128:ompthreads=128

#------------------------------------------------------------------------------
# END OF PBS DIRECTIVES
#------------------------------------------------------------------------------

echo "Job $PBS_JOBID started at `date` on `hostname`."

# Specify the ID string for the run. This can be set to any desired string.
# PBS_JOBNAME is used here as an example, as it is set by the #PBS -N
# directive near the top of this file.
runid=$PBS_JOBNAME

#------------------------------------------------------------------------------
# START OF MODULE DIRECTIVES
#------------------------------------------------------------------------------

# Load the required modules for serial kaiju.

# NOTE: This set of modules assumes your kaiju installation was built
# using this same list of modules. If you used different modules at
# build time (for example, if you used a GNU compiler), update this
# list to use the modules from your build-time environment.

# Comment out the module lines for all systems except the one you are using.
module purge

# For pleiades:
module load nas
module load pkgsrc/2022Q1-rome     # For git-lfs and cmake
module load comp-intel/2020.4.304  # Latest version
module load hdf5/1.8.18_serial

# For derecho:
# module load ncarenv/23.06
# module load craype/2.7.20
# module load intel/2023.0.0
# module load ncarcompilers/1.0.0  # Must come after intel/2023.0.0               
# module load hdf5/1.12.2
# module load cmake/3.26.3
# module load geos/3.9.1  # Must come after intel/2023.0.0                        

echo "The following modules are loaded:"
module list

#------------------------------------------------------------------------------
# END OF MODULE DIRECTIVES
#------------------------------------------------------------------------------

# Define the kaiju installation location.
# NOTE: You *must* set this variable to the path to your kaiju directory, which
# is the top-level directory created when you cloned the kaiju repository.
kaiju_install_dir=$HOME/kaiju

# This script sets KAIJUHOME and other environment variables.
source $kaiju_install_dir/scripts/setupEnvironment.sh

# Add the kaiju binary directory to the command path.
# NOTE: You *must* set this variable to the path to the bin subdirectory of
# your kaiju build directory. The setting below assumes that the serial version
# of kaiju was built in the build_serial subdirectory of the kaiju home
# directory (which is typically the same as kaiju_install_dir).
export PATH=$KAIJUHOME/build_serial/bin:$PATH

# Set the OMP stack size to prevent a crash.
# If this setting is ignored, the model may cause the kaiju code to crash
# with a segmentation fault and core dump. The value of "128M" was chosen
# ~arbitrarily; experimentation may allow a smaller value to be used.
# NOTE: OMP_STACKSIZE is the standard OpenMP environment variable for setting
# the OMP stack size. If using the Intel compiler, comment out the
# OMP_STACKSIZE line and uncomment the KMP_STACKSIZE line. KMP_STACKSIZE is the
# Intel-specific environment variable for setting the OMP stack size.
# export OMP_STACKSIZE=128M
export KMP_STACKSIZE=128M

echo "The active environment variables are:"
printenv

# Run the model. Direct output from the program is saved in a text file.
exe=gamera.x
echo "Running $exe on model $runid."
$exe ${runid}.xml >& ${exe}.${runid}.out

echo "Job $PBS_JOBID ended at `date` on `hostname`."
