#!/bin/bash

# This script was generated to run on {{ simulation.hpc_system }}.

#PBS -N {{ simulation.segment_id }}
#PBS -A {{ pbs.account_name }}
{%- if pbs.group_list|default('', true) and pbs.group_list != "None" %}
#PBS -W group_list={{ pbs.group_list }}
{%- endif %}
#PBS -q {{ pbs.queue }}
#PBS -l walltime={{ pbs.walltime }}
#PBS -l select={{ pbs.select }}:ncpus={{ pbs.ncpus }}:mpiprocs={{ pbs.mpiprocs }}:ompthreads={{ pbs.ompthreads }}{{ pbs.other }}+{{ pbs.select2 }}:ncpus={{ pbs.ncpus }}:mpiprocs={{ pbs.helper_mpiprocs }}:ompthreads={{ pbs.helper_ompthreads }}{{ pbs.other }}
{% if pbs.job_priority is defined %}#PBS -l job_priority={{ pbs.job_priority }}{% endif %}
#PBS -m abe
#PBS -j oe

echo "Job $PBS_JOBID started at `date` on `hostname`."

# Specify the ID string for the run.
export RUNID=$PBS_JOBNAME

# Some HPC systems do not start the job in the correct directory.
# Move to the intended directory. This will move the job to the correct
# directory on problem systems, such as pleiades (for some users), and have no
# effect on other systems.
cd $PBS_O_WORKDIR

# Load the required modules for kaiju.
module purge
{%- for moduledir in pbs.moduledir %}
module use -a {{ moduledir }}
{%- endfor %}
{%- for module in pbs.modules %}
module load {{ module }}
{%- endfor %}
module list

{%- for local_modules in pbs.local_modules %}
{{ local_modules }}
{%- endfor %}

# Define the kaiju installation location.
export KAIJU_INSTALL_DIR={{ pbs.kaiju_install_directory }}

# Set kaiju-related environment variables.
# This script sets KAIJUHOME and other environment variables.
source $KAIJU_INSTALL_DIR/scripts/setupEnvironment.sh

# Add the kaiju build bin directory to PATH.
export PATH={{ pbs.kaiju_build_directory }}/bin:$PATH

# Set other environment variables.
{{ pbs.additional_environment_variables -}}
export MPI_TYPE_DEPTH=32
export OMP_STACKSIZE=100M
export OMP_NUM_THREADS={{ pbs.helper_ompthreads }}

echo "The active environment variables are:"
printenv

# Copy the node file for use by the placer command.
nodefile=nodefile.$PBS_JOBID
cp $PBS_NODEFILE $nodefile

# Copy the main executable.
cp {{ pbs.kaiju_build_directory }}/bin/voltron_mpi.x ./voltron_mpi.x

# Run the model. Direct output from the program is saved in a text file.
EXE=./voltron_mpi.x
OUTPUT_FILE="$EXE-{{ simulation.segment_id }}.out"
echo "Running $EXE on model $RUNID."
{{ pbs.mpiexec_command }} $EXE $RUNID.xml &>> $OUTPUT_FILE

echo "Job $PBS_JOBID ended at `date` on `hostname`."
