writefile("funwave-tvd-docker/Dockerfile","""
FROM stevenrbrandt/science-base
USER root
RUN mkdir -p /home/install
RUN chown jovyan /home/install
USER jovyan

MAINTAINER Steven R. Brandt <sbrandt@cct.lsu.edu>
RUN cd /home/install && \
    git clone https://github.com/fengyanshi/FUNWAVE-TVD && \
    cd FUNWAVE-TVD/src && \
    perl -p -i -e 's/FLAG_8 = -DCOUPLING/#$&/' Makefile && \
    make

WORKDIR /home/install/FUNWAVE-TVD/src
RUN mkdir -p /home/jovyan/rundir
WORKDIR /home/jovyan/rundir
""")

"""
Explanation: <h2>Creating the Docker Image</h2>
To start, we need a Dockerfile, which has a number of simple commands.
It starts with "FROM" which can specify any docker image available from DockerHub. That not only includes basic operating systems such as "ubunto", "fedora", "centos", etc. but specialized containers made by anyone with a dockerhub account. I've provided "science-base" which has OpenMPI 2.1.1 and some standard compilers, i.e. gfortran, gcc, and g++.
MAINTAINER is a bit of metadata that (hopefully) will allow you to contact the container's creator, if need be.
WORKDIR is the dockerfile equivalent of the "cd" command. Note that running "cd" will not change your directory.
RUN simply runs the command that follows. Because the container is saved after each step, we want to avoid creating files that we don't want to keep (we want containers to be as small as possible).
USER specifies the user id for running subsequent RUN commands.
COPY can be used to copy files into the container from the build directory.
ENTRYPOINT is a script that runs when the container starts up. What our script does is create a new user on the docker image with a user id and name that is convenient.
End of explanation
"""


!tar -czf dockerjob.tgz -C funwave-tvd-docker Dockerfile
!files-mkdir -S ${AGAVE_STORAGE_SYSTEM_ID} -N funwave-tvd-docker
!files-upload -F dockerjob.tgz -S ${AGAVE_STORAGE_SYSTEM_ID} funwave-tvd-docker/

import runagavecmd as r
import imp
imp.reload(r)

"""
Explanation: Now that we've create our Dockerfile and entrypoint.sh, bundle them up in a tarball and send them somewhere that agave can access them.
End of explanation
"""


r.runagavecmd(
    "tar xzf dockerjob.tgz && sudo docker build --rm -t funwave-tvd-2 .",
    "agave://${AGAVE_STORAGE_SYSTEM_ID}/funwave-tvd-docker/dockerjob.tgz"
)

!jobs-output-get ${JOB_ID} fork-command-1.err
!cat fork-command-1.err

"""
Explanation: Run the docker build command. We will "tag" this build with the name "funwave-tvd" when it is complete.
End of explanation
"""


writefile("rundock.sh","""
rm -fr cid.txt out.tgz

# Start a docker image running in detached mode, write the container id to cid.txt
sudo docker run -d -it --rm --cidfile cid.txt funwave-tvd-2 bash

# Store the container id in CID for convenience
CID=\$(cat cid.txt)

# Copy the input.txt file into the running image
sudo docker cp input.txt \$CID:/home/jovyan/rundir/

# Run funwave on the image
sudo docker exec --user jovyan \$CID mpirun -np 2 /home/install/FUNWAVE-TVD/src/funwave_vessel

# Extract the output files from the running image
# Having them in a tgz makes it more convenient to fetch them with jobs-output-get
sudo docker exec --user jovyan \$CID tar czf - output > out.tgz

# Stop the image
sudo docker stop \$CID

# List the output files
tar tzf out.tgz
""")

"""
Explanation: <h2>Running the Docker Image</h2>
It is possible to run docker interactively, but that isn't convenient inside scripts. So instead, we start it in detached mode, with the -d flag.
Because your docker image has its own internal file system, it can't see files on the host machine. You can, however, transfer them using the "docker cp" command.
Running docker is slightly tricky. When a Docker image starts up, you can execute any command you want--but when you type "exit" all the changes you've made to the file system vanish. Therefore it's necessary to copy them out before the docker container stops.
End of explanation
"""


!tar czf rundock.tgz rundock.sh input.txt
!files-upload -F rundock.tgz -S ${AGAVE_STORAGE_SYSTEM_ID} funwave-tvd-docker/

"""
Explanation: Upload the input.txt file and the rundock.sh script.
End of explanation
"""


r.runagavecmd(
    "tar xzf rundock.tgz && bash rundock.sh",
    "agave://${AGAVE_STORAGE_SYSTEM_ID}/funwave-tvd-docker/rundock.tgz")

"""
Explanation: Execute the rundock.sh script
End of explanation
"""


!jobs-output-list ${JOB_ID}
!jobs-output-get ${JOB_ID} out.tgz
!tar xzf out.tgz

!head output/eta_00010

"""
Explanation: Get the output of the job back to our local machine
End of explanation
"""


!files-mkdir -S ${AGAVE_STORAGE_SYSTEM_ID} -N sing
!files-upload -F input.txt -S ${AGAVE_STORAGE_SYSTEM_ID} sing/
r.runagavecmd(
            "mkdir -p ~/singu && "+
            "cd ~/singu && "+
            "rm -f funwave-tvd.img && "+
            "singularity create funwave-tvd.img --size 2000 && "+
            "singularity import funwave-tvd.img docker://stevenrbrandt/funwave-tvd-2:latest")

"""
Explanation: <h2>Running with Singularity</h2>
If we have a public docker image, we can run it directly with Singularity. Singularity is desiged to be more HPC friendly than Docker. First, because it doesn't all the running user to access any user id but their own inside the container, and second, because singularity images can be run through MPI, making it easier to scale up to a distributed cluser.
In this first step, we build the singularity installation. Because the result of this job is intended to be an installation for subsequent jobs, we install it to a hard-coded directory rather than using the normal Agave job directory.
End of explanation
"""


!files-upload -F input.txt -S ${AGAVE_STORAGE_SYSTEM_ID} ./
r.runagavecmd(
    "export LD_LIBRARY_PATH=/usr/local/lib && "+
    "mpirun -np 2 singularity exec ~/singu/funwave-tvd.img /home/install/FUNWAVE-TVD/src/funwave_vessel && "+
    "tar cvzf singout.tgz output",
    "agave://${AGAVE_STORAGE_SYSTEM_ID}/input.txt"
)

!jobs-output-get ${JOB_ID} singout.tgz
!rm -fr output
!tar xzf singout.tgz

!head output/v_00003

"""
Explanation: Now that the Singularity image is built, we can run it with mpi. Notice that mpi executes the singularity command. The tricky part here is to make sure you've got the same version of mpi running inside and outside the container.
End of explanation
"""


!echo /home/jovyan/singu/funwave-tvd.img > ~/work/sing.txt

"""
Explanation: In this next and final singularity example, we get around the problem of needing to port MPI by using the same MPI that's in the container to launch the containers. The trick is this code, which comes at the end of the .bashrc. What it does is effectively replace our shell on the machine with an execution of bash inside the singularity image.
<pre>
# Put the full path to a singularity image in the file $HOME/sing.txt.
if [ -r $HOME/work/sing.txt ]
then
    IMAGE=$(cat $HOME/work/sing.txt)
fi
if [ "$IMAGE" != "" ]
then
    if [ -r "$IMAGE" ]
    then
        # If the SINGULARITY_CONTAINER variable is set,
        # then we are already in the container
        if [ "$SINGULARITY_CONTAINER" = "" ]
        then
            # Switch to running inside singularity
            exec singularity exec $IMAGE bash --login
        fi
    else
        echo Could not read image file $IMAGE
    fi
fi
</pre>
End of explanation
"""


!files-upload -F input.txt -S ${AGAVE_STORAGE_SYSTEM_ID} ./
r.runagavecmd(
    "rm -fr output && "+
    "mpirun -np 2 /home/install/FUNWAVE-TVD/src/funwave_vessel && "+
    "tar cvzf singout.tgz output",
    "agave://${AGAVE_STORAGE_SYSTEM_ID}/input.txt"
)

!rm -fr output singout.tgz
!jobs-output-get ${JOB_ID} singout.tgz
!tar xzf singout.tgz
!ls output

# Clean up so that we don't boot into the singularity image without intending to
!rm -f ~/work/sing.txt

"""
Explanation: There's no need to call singularity explicitly, as it's called by each invocation of .bashrc. Note that the funwave-tvd we are executing is the one from inside the image.
End of explanation
"""


