"""

"""

from collections import Counter
import numpy as np
import json
import csv


if __name__ == "__main__":
    with open("data/arxivData.json") as f:
        arxiv_data = json.load(f)

    print("Found {} samples.".format(len(arxiv_data)))

    abstract_lengths = []
    title_lengths = []
    for paper in arxiv_data:
        abstract_lengths.append(len(paper["summary"]))
        title_lengths.append(len(paper["title"]))

    minval = 0
    maxval = max(abstract_lengths)
    bins = np.arange(minval, maxval+50, 50)
    histogram, edges = np.histogram(abstract_lengths, bins)

    # anything between 200 - 2000 characters seems reasonable
    for lo, hi, num in zip(edges, edges[1:], histogram):
        print("{} - {}: {}".format(lo, hi, num))

    minval = 0
    maxval = max(title_lengths)
    bins = np.arange(minval, maxval+5, 5)
    histogram, edges = np.histogram(title_lengths, bins)

    # 10 - 200 seems reasonable here (?)
    for lo, hi, num in zip(edges, edges[1:], histogram):
        print("{} - {}: {}".format(lo, hi, num))


    with open("data/Donald-Tweets!.csv") as f:
        lines = f.readlines()

    # skip header manually
    lines = lines[1:]
    tweet_lengths = []
    for i, line in enumerate(csv.reader(lines, quotechar='"', delimiter=",")):
        tweet_lengths.append(len(line[2]))

    minval = 0
    maxval = max(tweet_lengths)
    bins = np.arange(minval, maxval+5, 5)
    histogram, edges = np.histogram(tweet_lengths, bins)

    # 10 - 200 seems reasonable here (?)
    for lo, hi, num in zip(edges, edges[1:], histogram):
        print("{} - {}: {}".format(lo, hi, num))



