#!/bin/bash

# This script was generated to run on {{ simulation.hpc_system }}.

#PBS -N {{ simulation.segment_id }}
#PBS -A {{ pbs.account_name }}
{%- if pbs.group_list|default('', true) and pbs.group_list != "None" %}
#PBS -W group_list={{ pbs.group_list }}
{%- endif %}
#PBS -q {{ pbs.queue }}
#PBS -l walltime={{ pbs.walltime }}
{%- if pbs.model is defined %}
#PBS -l select={{ pbs.tie_nodes }}:ncpus={{ pbs.tie_ncpus }}:mpiprocs={{ pbs.tie_mpiprocs }}:model={{ pbs.model }}+{{ pbs.gamera_nodes }}:ncpus={{ pbs.gamera_ncpus }}:mpiprocs={{ pbs.gamera_mpiprocs }}:ompthreads={{ pbs.gamera_ompthreads }}:model={{ pbs.model }}+{{ pbs.voltron_nodes }}:ncpus={{ pbs.voltron_ncpus }}:mpiprocs={{ pbs.voltron_mpiprocs }}:ompthreads={{ pbs.voltron_ompthreads }}:model={{ pbs.model }}
{%- else %}
#PBS -l select={{ pbs.tie_nodes }}:ncpus={{ pbs.tie_ncpus }}:mpiprocs={{ pbs.tie_mpiprocs }}+{{ pbs.gamera_nodes }}:ncpus={{ pbs.gamera_ncpus }}:mpiprocs={{ pbs.gamera_mpiprocs }}:ompthreads={{ pbs.gamera_ompthreads }}+{{ pbs.voltron_nodes }}:ncpus={{ pbs.voltron_ncpus }}:mpiprocs={{ pbs.voltron_mpiprocs }}:ompthreads={{ pbs.voltron_ompthreads }}
{%- endif %}
{%- if pbs.job_priority is defined %}
#PBS -l job_priority={{ pbs.job_priority }}
{%- endif %}
#PBS -m abe
#PBS -j oe

echo "Job $PBS_JOBID started at `date` on `hostname`."

# Specify the ID string for the run.
export RUNID=$PBS_JOBNAME

# Some HPC systems do not start the job in the correct directory.
# Move to the intended directory. This will move the job to the correct
# directory on problem systems, such as pleiades (for some users), and have no
# effect on other systems.
cd $PBS_O_WORKDIR

# Load the required modules for kaiju.
module purge
{%- for moduledir in pbs.moduledir %}
module use -a {{ moduledir }}
{%- endfor %}
{%- for module in pbs.modules %}
module load {{ module }}
{%- endfor %}
module list

{%- for local_modules in pbs.local_modules %}
{{ local_modules }}
{%- endfor %}

# Define the kaiju installation location.
export KAIJU_INSTALL_DIR={{ pbs.kaiju_install_directory }}

# Set kaiju-related environment variables.
# This script sets KAIJUHOME and other environment variables.
source $KAIJU_INSTALL_DIR/scripts/setupEnvironment.sh

# Add the kaiju build bin directory to PATH.
export PATH={{ pbs.kaiju_build_directory }}/bin:$PATH

# Set other environment variables.
{{ pbs.additional_environment_variables -}}
export MPI_TYPE_DEPTH=32
export OMP_STACKSIZE=100M
export OMP_NUM_THREADS={{ pbs.helper_ompthreads }}

echo "The active environment variables are:"
printenv

# Copy the node file for use by the placer command.
NODEFILE=nodefile.$PBS_JOBID
cp $PBS_NODEFILE $NODEFILE

{%- if pbs.nodecommand is defined %}
{{ pbs.nodecommand }}
{%- endif %}

# Copy the main executable.
# cp {{ pbs.kaiju_build_directory }}/bin/voltron_mpi.x ./voltron_mpi.x
# cp {{ pbs.tie_exe }} ./tiegcm.x

# Run the model. Direct output from the program is saved in a text file.
VOLTRON_EXE=./voltron_mpi.x
TIEGCM_EXE=./tiegcm.x
OUTPUT_FILE="$VOLTRON_EXE-{{ simulation.segment_id }}.out"
echo "Running $VOLTRON_EXE on model $RUNID."
{{ pbs.mpiexec_command }} {{pbs.mpiexec_option}} {{pbs.tie_mpiranks}} {%- if pbs.tie_scripts is defined %} {{ pbs.tie_scripts }} {%- endif %} $TIEGCM_EXE {{pbs.tie_inp}} : {{pbs.mpiexec_option}} {{pbs.voltron_mpiranks}} {%- if pbs.voltron_scripts is defined %} {{ pbs.voltron_scripts }} {%- endif %} $VOLTRON_EXE $RUNID.xml &>> $OUTPUT_FILE

echo "Job $PBS_JOBID ended at `date` on `hostname`."
