To run Alphafold on Ibex, the following guide is recommended. To start with, you need to pull a container image from KAUST RCCL registry:

module load singularity
export TMPDIR=$HOME/tmpdir
mkdir -p $TMPDIR

singularity pull docker://krccl/alphafold:2.3.2

Batch job

#!/bin/bash
#SBATCH -N 1
#SBATCH -J AlphaFold.version2.3.1
#SBATCH --time=10:0:0
#SBATCH --mem=100G
#SBATCH --gpus=1
#SBATCH --gpus-per-node=1
#SBATCH --cpus-per-task=40
#SBATCH --constraint=v100


module load singularity

export IMAGE=./alphafold_2.3.2.sif
export ALPHAFOLD_DATA_DIR=/ibex/reference/KSL/alphafold/2.3.1

export OUTDIR=${PWD}/output
mkdir -p $OUTDIR

# Start NVDASHBOARD for monitoring GPU usage. Please comment this section if not needed.
# get tunneling info 
export XDG_RUNTIME_DIR=/tmp node=$(hostname -s) 
user=$(whoami) 
submit_host=${SLURM_SUBMIT_HOST} 
nv_port=$(python -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()')

singularity run $IMAGE nvdashboard ${nv_port} &
sleep 5 

# print tunneling instructions
echo -e "
To connect to the nvdashboard session, you need to run following  command in a new terminal 

ssh -L ${nv_port}:${node}.ibex.kaust.edu.sa:${nv_port} ${user}@glogin.ibex.kaust.edu.sa


# Launch Alphafold

srun -c $SLURM_CPUS_PER_TASK singularity run --env \
  TF_FORCE_UNIFIED_MEMORY=1,XLA_PYTHON_CLIENT_MEM_FRACTION=5.0,OPENMM_CPU_THREADS=${SLURM_CPUS_PER_TASK} \
  -B .:/etc -B $ALPHAFOLD_DATA_DIR:/data \
  --pwd /app/alphafold \
  --nv $IMAGE  \
  /app/run_alphafold.sh \
  --use_gpu_relax=True \
  --max_template_date=2020-05-14 \
  --fasta_paths=${PWD}/WT_WT.fa  \
  --data_dir=/data \
  --uniref90_database_path=/data/uniref90/uniref90.fasta \
  --uniref30_database_path=/data/uniref30/UniRef30_2021_03 \
  --output_dir=${OUTDIR} \
  --mgnify_database_path=/data/mgnify/mgy_clusters_2022_05.fa \
  --template_mmcif_dir=/data/pdb_mmcif/mmcif_files \
  --obsolete_pdbs_path=/data/pdb_mmcif/obsolete.dat \
  --bfd_database_path=/data/bfd/bfd_metaclust_clu_complete_id30_c90_final_seq.sorted_opt \
  --model_preset=multimer \
  --pdb_seqres_database_path=/data/pdb_seqres/pdb_seqres.txt \
  --uniprot_database_path=/data/uniprot

Interactive Jupyter Session

#!/bin/bash
#SBATCH -N 1
#SBATCH -J AlphaFold.version2.3.1
#SBATCH --time=0:30:0
#SBATCH --mem=100G
#SBATCH --gpus=1
#SBATCH --gpus-per-node=1
#SBATCH --cpus-per-task=8
#SBATCH --constraint=v100



nvidia-smi --query-gpu=timestamp,utilization.gpu,utilization.memory,memory.used --format=csv -l 10 >> monitor.log &


module load singularity

export IMAGE=./alphafold_2.3.2.sif

export ALPHAFOLD_DATA_DIR=/ibex/reference/KSL/alphafold/2.3.1



# setup ssh tunneling
# get tunneling info 
export XDG_RUNTIME_DIR=/tmp node=$(hostname -s) 
user=$(whoami) 
submit_host=${SLURM_SUBMIT_HOST} 
port=$(python -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()')
nv_port=$(python -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()')


echo ${node} pinned to port ${port} on ${submit_host} 

singularity run $IMAGE nvdashboard ${nv_port} &

sleep 5 

# print tunneling instructions
echo -e "
${node} pinned to port ${port} on ${submit_host}
To connect to the compute node ${node} on IBEX running your jupyter notebook server, you need to run following two commands in a terminal 1.
Command to create ssh tunnel from you workstation/laptop to glogin:

ssh -L ${port}:${node}.ibex.kaust.edu.sa:${port} -L ${nv_port}:${node}.ibex.kaust.edu.sa:${nv_port} ${user}@glogin.ibex.kaust.edu.sa

Copy the link provided below by jupyter-server and replace the NODENAME with localhost before pasting it in your browser on your workstation/laptop.
" >&2

srun -c $SLURM_CPUS_PER_TASK singularity run --env \
  TF_FORCE_UNIFIED_MEMORY=1,XLA_PYTHON_CLIENT_MEM_FRACTION=5.0,OPENMM_CPU_THREADS=${SLURM_CPUS_PER_TASK} \
  PYTHONPATH=/app/alphafold \
  -B .:/etc -B $ALPHAFOLD_DATA_DIR:/data \
  --nv $IMAGE  \
  jupyter lab --no-browser --port=${port} --port-retries=0  --ip=0.0.0.0

For connecting to the Jupyter session, there are two steps:

Copy paste the ssh -L ... line for the output file slurm-######.out and paste it in a new terminal window
Copy paste the http://127.0.0.1:..... line from the slurm-#####.out file, including the token part, and paste it in your local web browser.

KSL How-To repository

Google's AlphaFold2 on Ibex

Batch job

Interactive Jupyter Session