#!/bin/bash
#PBS -A FUS117
#PBS -l walltime=0:05:00
#PBS -l nodes=2
##PBS -l procs=1
##PBS -l gres=atlas1%atlas2


#Cannot see home folder, will just hang until wall limit
export HOME=/ccs/proj/fus117/
cd $HOME/PPPL/plasma-python/examples

source $MODULESHOME/init/bash
module switch PrgEnv-pgi PrgEnv-gnu

module load cudatoolkit
export LIBRARY_PATH=/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/lib64:$LIBRARY_PATH

#This block is CuDNN module
export LD_LIBRARY_PATH=$HOME/cuda/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$HOME/cuda/lib64:$LIBRARY_PATH
export LDFLAGS=$LDFLAGS:$HOME/cuda/lib64
export INCLUDE=$INCLUDE:$HOME/cuda/include
export CPATH=$CPATH:$HOME/cuda/include
export FFLAGS=$FFLAGS:$HOME/cuda/include
export LOCAL_LDFLAGS=$LOCAL_LDFLAGS:$HOME/cuda/lib64
export LOCAL_INCLUDE=$LOCAL_INCLUDE:$HOME/cuda/include
export LOCAL_CFLAGS=$LOCAL_CFLAGS:$HOME/cuda/include
export LOCAL_FFLAGS=$LOCAL_FFLAGS:$HOME/cuda/include
export LOCAL_CXXFLAGS=$LOCAL_CXXFLAGS:$HOME/cuda/include

#This sets new home and Anaconda module
export PATH=$HOME/anaconda2/bin:$PATH
export LD_LIBRARY_PATH=$HOME/anaconda2/lib:$LD_LIBRARY_PATH
source activate PPPL

PYTHON=`which python`
echo $PYTHON

#pygpu backend
#export CPATH=$CPATH:~/.local/include
#export LIBRARY_PATH=$LIBRARY_PATH:~/.local/lib
#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:~/.local/lib

export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH
export MPICH_RDMA_ENABLED_CUDA=1

rm $HOME/tigress/alexeys/model_checkpoints/*
aprun -n2 -N1 $PYTHON mpi_learn.py