# install kipoiseq
# !pip install kipoiseq

from kipoiseq.dataloaders import SeqIntervalDl

"""
Explanation: Getting started
End of explanation
"""


kwargs = SeqIntervalDl.example_kwargs
kwargs

!cat {kwargs['fasta_file']} | head -c 100

!cat {kwargs['intervals_file']}

"""
Explanation: Get the example files
SeqDaset comes with some example files that get downloaded
End of explanation
"""


# setup the dataset
dl = SeqIntervalDl(**kwargs)

len(dl)

dl[0]

"""
Explanation: Extra columns (all ones) are binary labels for the interval
Setup the dataset
End of explanation
"""


len(dl[0]['inputs'])

"""
Explanation: Sequence is one-hot encoded. You can see that the first interval matches the one provided in the intervals file. You can also see that the extra columns got parsed as targets.
End of explanation
"""


dl = SeqIntervalDl(auto_resize_len=10, **kwargs)

len(dl[0]['inputs'])

"""
Explanation: Since the intervals are all of variable length, we have to resize them.
End of explanation
"""


data = dl.load_all()

import pandas as pd

pd.DataFrame(data['metadata']['ranges'])

"""
Explanation: You can load the whole dataset into memory
End of explanation
"""


# setup a simple model
import keras.layers as kl
from keras.models import Sequential

model = Sequential([kl.Conv1D(3, 2, activation='relu', input_shape=(10,4)), 
                    kl.GlobalMaxPool1D(), 
                    kl.Dense(1)])

model.compile('adam', 'binary_crossentropy', ['acc'])

batch_size = 2

# setup an iterator
iterator = dl.batch_train_iter(batch_size=batch_size, num_workers=4)   # use 4 workers in paralellel to load the data

model.fit_generator(iterator, steps_per_epoch=len(dl)//batch_size, epochs=10)

"""
Explanation: Or you can load it batch by batch
Training a Keras model
End of explanation
"""


x,y = next(iterator)

x  # one-hot encoded DNA sequence

x.shape

y  # target labels

y.shape

"""
Explanation: What does the iterator return?
End of explanation
"""


from kipoi.writers import HDF5BatchWriter

writer = HDF5BatchWriter("/tmp/preds.h5")

for batch in dl.batch_iter(batch_size=batch_size, num_workers=4):
    preds = model.predict_on_batch(batch['inputs'])
    to_write = {"preds": preds, "metadata": batch['metadata']}
    writer.batch_write(to_write)

writer.close()

"""
Explanation: Making predictions and writing them iteratively to an hdf5 file
Let's say you have a very large dataset and you want to save the predictions batch-by-batch into an hdf5 file together with the metadata. Here is how you can do that
End of explanation
"""


from kipoi.readers import HDF5Reader

reader = HDF5Reader('/tmp/preds.h5')
reader.open()

# list all the arrays
reader.ls()

data = reader.load_all()

data['preds']

# handle to the h5py objec
reader.f

reader.f['preds'][:2]

"""
Explanation: Let's have a look at what we wrote
End of explanation
"""


