get_ipython().magic('load_ext cellevents')
get_ipython().magic('load_ext autoreload')
get_ipython().magic('autoreload 2')
from logcon import log
from xdrive import aws, server, apps
from xdrive.drive import Drive
import fabric.api as fab
from fabric.state import connections
apps.setdebug()

"""
Explanation: Overview of xdrive package
This package puts programs and data on a portable "xdrive" rather than an on 
the AWS instance boot drive. The "xdrive" can then be moved between different types of 
server including spot instances. Ths saves 100% of the cost of setting up data and programs by using free tier
servers; and 80% of the cost of GPUs by providing persistent storage for spot
instances. The examples here show how to setup and work with various types of server.
Note that xdrive holds minimal state so you can continue to use AWS menus in
parallel.
Imports
End of explanation
"""


# create a key
import os
keyfile = os.path.join(os.path.expanduser("~"), ".aws/key.pem")
try:
    key = aws.ec2.create_key_pair(KeyName="key")
    with open(keyfile, "w") as f:
        f.write(key.key_material)
except Exception as e:
    log.warning(e)

# create a security group
try:
  sec = aws.ec2.create_security_group(GroupName="simon", 
                                Description="wordpress, jupyter, ssh")
  sec.authorize_ingress(
      IpPermissions=[dict(IpProtocol='tcp', FromPort=80, ToPort=80),
                     dict(IpProtocol='tcp', FromPort=443, ToPort=443),
                     dict(IpProtocol='tcp', FromPort=8888, ToPort=8888),
                     dict(IpProtocol='tcp', FromPort=22, ToPort=22)])
except Exception as e:
    log.warning(e)

"""
Explanation: Configuration
End of explanation
"""


server.create("kate", itype="free", drive="fastai", drivesize=15)

"""
Explanation: Setup programs and data using a free instance
Create a server called "kate" with a free instance; and a xdrive called "fastai" mounted at /v1 to hold programs and data.
Note before SSH to server wait until cell finishes. Otherwise you may not have correct permissions for docker.
End of explanation
"""


apps.run_fastai()

"""
Explanation: Download stuff via ssh
It can take several minutes to pull a large docker image or data file; and doing this via a notebook either produces excessive output or a silent wait. Therefore carry out these steps via SSH as it is easier to monitor progress. 
* download data to /v1 (as required)
* docker pull simonm3/fastai
Test the notebook
Once the docker pull has completed you can run the next cell which runs the notebook. You can then test the container is working by viewing notebooks at the "kate" ip address port 8888 with password is dl_course.
Note this notebook CANNOT be ported to the GPU
Containers can be ported between GPUs. However you cannot directly port a container from a CPU to a GPU. The only way to do this is to commit it as an image; and then run a new container based on that image. This is generally not a big deal as most of the processing will be carried out on GPUs; and you can port between GPU spot instances.
End of explanation
"""


fab.run("docker rm -f fastai")
server.terminate("kate")

"""
Explanation: All of the setup time so far has used free instances and free storage. Next step is to delete the container and terminate the instance. We will create a new container on the GPU.
End of explanation
"""


server.create("sarah", itype="gpu", spotprice=.3, drive="fastai")

"""
Explanation: Work with the programs and data using a GPU
Create a spot GPU server called "sarah" with the same data and programs as before. Note wait for the cell to complete before attempting SSH.
End of explanation
"""


apps.run_fastai()
#apps.start_fastai()

"""
Explanation: If this is the first time running on GPU then run_fastai() creates a new container. Subsequently start_fastai() starts the existing container which retains all the settings from the last run. Once the notebook is available the ip address will be in the clipboard so you just ctrl-v into the browser address bar. Again, wait for the cell to complete as it will tell you when the notebook is available which can take a minute or so.
End of explanation
"""


server.terminate("sarah")

"""
Explanation: When you have finished working then call server.terminate("sarah"). This will saves the xdrive as a snapshot including all data and programs. When a spot instance is outbid then AWS sends a 2 minute termination notice. This will be captured and result in a call to server.terminate.
End of explanation
"""


instance = server.create("sm")
aws.associate_address("sm")
server.wait_ssh()
apps.install_docker()
fab.sudo("service docker start")
apps.install_wordpress()

"""
Explanation: Create more servers
It is possible to create servers without a xdrive. For example you may want to create a server with a static IP address running wordpress.
* request an elastic ip address from AWS (this is free as long as attached to a running instance)
* run script below. 
* associate_address takes parameter ip which defaults to first elastic ip on account
End of explanation
"""


xdrive = Drive("fastai")
xdrive.connect("sm")

xdrive.disconnect()

"""
Explanation: Work with an existing xdrive
Typically you will create a server with an attached xdrive. However sometimes you may want to attach the xdrive to an existing instance. This is also possible with the commands below. Once connected you can work with /v1 starting docker if you need to. When you have finished then just call disconnect.
End of explanation
"""


# get a resource by name
aws.get("sm")

# get all resources (instances, volumes, snapshots)
aws.get(unique=False)

# show instances used
aws.get_instances()

# show python tasks running in containers
fab.env.host_string=aws.get("sm").public_ip_address
server.get_tasks("python")

# install python app in a container including config files from laptop
apps.install_python("meetups", configs=[".meetups.yaml", ".gmail.yaml"])

"""
Explanation: Utilities
As a bonus there are a number of utilities available as below. Also, for convenience, all resources (instances, volumes, snapshots) can be referred to by name rather than the amazon 20 character id.
End of explanation
"""


