# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import os
import tarfile
import urllib.request
import sys
import glob
from bs4 import BeautifulSoup
import nltk
from string import ascii_lowercase

import tensorflow as tf

TEMP_DIR = '/tmp/tensorflow_tutorials'
WORD_CHARS = set(ascii_lowercase + "'!?-.()")

def download_and_cache(url, fname=None, dest=TEMP_DIR):
    if not os.path.exists(dest):
        os.makedirs(dest)
    if fname is None:
        fname = url.split('/')[-1]
    fpath = os.path.join(dest, fname)
    if not os.path.exists(fpath):
        def _progress(count, block_size, total_size):
            percentage = float(count * block_size) / float(total_size) * 100.0
            sys.stdout.write('\r>> Downloading {} {:1.1f}%'.format(fname, percentage))
            sys.stdout.flush()
        fpath, _ = urllib.request.urlretrieve(url, fpath, _progress)
        print()
        statinfo = os.stat(fpath)
        print('Successfully downloaded', fname, statinfo.st_size, 'bytes.')
    return fpath
    


# -

fpath = download_and_cache('http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz')

with tarfile.open(fpath, 'r:gz') as tar:
    tar.extractall(TEMP_DIR)

train_pos = glob.glob(os.path.join(TEMP_DIR, 'aclImdb', 'train/pos/', '*.txt'))
train_neg = glob.glob(os.path.join(TEMP_DIR, 'aclImdb', 'train/neg/', '*.txt'))

filenames = train_pos + train_neg
labels = [1]*len(train_pos) + [0]*len(train_neg)

dataset = tf.data.Dataset.from_tensor_slices((filenames, labels))


# +
def _is_word(word):
    return set(word.lower()).issubset(WORD_CHARS)

def _preprocess_text(input_text, label):
    soup = BeautifulSoup(input_text, "lxml")
    sents = nltk.sent_tokenize(soup.get_text())
    words = [nltk.word_tokenize(sent) for sent in sents]
    res = ' '.join(' '.join(word.lower() for word in sent_word if _is_word(word)) for sent_word in words)
    return res, label
    
def _read_files(filename, label):
    file_content = tf.read_file(filename)
    return file_content, label


# -

dataset = dataset.map(_read_files)


def wrapped_func(text, label):
    return tuple(tf.py_func(_preprocess_text, [text, label], [tf.string, label.dtype]))
dataset = dataset.map(wrapped_func)

iterator = dataset.make_one_shot_iterator()
next_element = iterator.get_next()

with tf.Session() as sess:
    res = sess.run(next_element)

print(res)

with open(filenames[0], 'r') as fp:
    print(fp.readlines())

# +
# Other resources:
# https://cs230-stanford.github.io/tensorflow-input-data.html
# -



StarcoderdataJupyterScriptsDedupFiltered