import json
import re
import sys


def beautify(name):
    ''' Loading, filtering and saving the JSON tweet file to a newly generated .txt file
    :type: name: String
    :rtype: output: .txt
    '''

    filename = name + '.json'
    output_name = name + "_filtered.txt"

    with open(filename, "r", encoding="utf-8") as input:
        with open(output_name, "w", encoding="utf-8") as output:
            document = json.load(input)
            
            # Filter only the messages that are not retweeted
            # >> Version i): for tweets from archive "master_XXXX.json" 
            # document = [x['full_text'] for x in document if x['user']['screen_name'] == 'realDonaldTrump' and 'full_text' in x]
            
            # >> Version ii): for self-scraped tweets via https://github.com/bpb27/twitter_scraping
            # document = [x['text'] for x in document if x['user']['screen_name'] == 'realDonaldTrump' and 'text' in x]

            # >> Version iii): Data set from https://github.com/MatthewWolff/MarkovTweets/
            document = [x['text'] for x in document]

            # Clean and only include not retweeted messages
            document = [deep_clean(x) for x in document if deep_clean(x) is not None]

            # Preventing unicode characters by ensuring false ascii encoding
            for _, value in enumerate(document):
                output.write(json.dumps(value, ensure_ascii=False) + "\n")
            # json.dump(document, output, ensure_ascii=False, indent=4)

    print(f">> Sucessfully cleaned {filename} and saved it to {output_name}")


def deep_clean(s):
    ''' Deep cleaning of filtered tweets. Replaces common symbols and kills quotation marks/apostrophes.
    :type: s: String
    :rtype: s: String
    '''

    # Return None if given tweet is a retweet
    if s[:2] == 'RT':
        return None

    # Delete all URLs because they don't make for interesting tweets.
    s = re.sub(r'http[\S]*', '', s)

    # Replace some common unicode symbols with raw character variants
    s = re.sub(r'\\u2026', '...', s)
    s = re.sub(r'…', '', s)
    s = re.sub(r'\\u2019', "'", s)
    s = re.sub(r'\\u2018', "'", s)
    s = re.sub(r"&amp;", r"&", s)
    s = re.sub(r'\\n', r"", s)

    # Delete emoji modifying characters
    s = re.sub(chr(127996), '', s)
    s = re.sub(chr(65039), '', s)

    # Kill apostrophes & punctuation because they confuse things.
    s = re.sub(r"'", r"", s)
    s = re.sub(r"“", r"", s)
    s = re.sub(r"”", r"", s)
    s = re.sub('[()]', r'', s)
    s = re.sub(r'"', r"", s)

    # Collapse multiples of certain chars
    s = re.sub('([.-])+', r'\1', s)
    
    # Pad sentence punctuation chars with whitespace
    s = re.sub('([^0-9])([.,!?])([^0-9])', r'\1 \2 \3', s)
    
    # Remove extra whitespace (incl. newlines)
    s = ' '.join(s.split()).lower()

    # Define emoji_pattern
    emoji_pattern = re.compile("["
        u"\U0001F600-\U0001F64F"  # emoticons
        u"\U0001F300-\U0001F5FF"  # symbols & pictographs
        u"\U0001F680-\U0001F6FF"  # transport & map symbols
        u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
        u"\U0001F1F2-\U0001F1F4"  # Macau flag
        u"\U0001F1E6-\U0001F1FF"  # flags
        u"\U0001F600-\U0001F64F"
        u"\U00002702-\U000027B0"
        u"\U000024C2-\U0001F251"
        u"\U0001f926-\U0001f937"
        u"\U0001F1F2"
        u"\U0001F1F4"
        u"\U0001F620"
        u"\u200d"
        u"\u2640-\u2642"
        "]+", flags=re.UNICODE)

    s = emoji_pattern.sub(r'', s)

    # Care for a special case where the first char is a "."
    # return s[1:] if s[0] == "." else s
    if len(s):
        return s[1:] if s[0] == "." else s
    return None


if __name__ == "__main__":

    if len(sys.argv) - 1: beautify(sys.argv[1]) 
[((51, 8, 51, 35), 're.sub', 're.sub', ({(51, 15, 51, 27): '"""http[\\\\S]*"""', (51, 29, 51, 31): '""""""', (51, 33, 51, 34): 's'}, {}), "('http[\\\\S]*', '', s)", False, 'import re\n'), ((54, 8, 54, 36), 're.sub', 're.sub', ({(54, 15, 54, 25): '"""\\\\\\\\u2026"""', (54, 27, 54, 32): '"""..."""', (54, 34, 54, 35): 's'}, {}), "('\\\\\\\\u2026', '...', s)", False, 'import re\n'), ((55, 8, 55, 29), 're.sub', 're.sub', ({(55, 15, 55, 21): '"""…"""', (55, 23, 55, 25): '""""""', (55, 27, 55, 28): 's'}, {}), "('…', '', s)", False, 'import re\n'), ((56, 8, 56, 34), 're.sub', 're.sub', ({(56, 15, 56, 25): '"""\\\\\\\\u2019"""', (56, 27, 56, 30): '"""\'"""', (56, 32, 56, 33): 's'}, {}), '(\'\\\\\\\\u2019\', "\'", s)', False, 'import re\n'), ((57, 8, 57, 34), 're.sub', 're.sub', ({(57, 15, 57, 25): '"""\\\\\\\\u2018"""', (57, 27, 57, 30): '"""\'"""', (57, 32, 57, 33): 's'}, {}), '(\'\\\\\\\\u2018\', "\'", s)', False, 'import re\n'), ((58, 8, 58, 33), 're.sub', 're.sub', ({(58, 15, 58, 23): '"""&amp;"""', (58, 25, 58, 29): '"""&"""', (58, 31, 58, 32): 's'}, {}), "('&amp;', '&', s)", False, 'import re\n'), ((59, 8, 59, 30), 're.sub', 're.sub', ({(59, 15, 59, 21): '"""\\\\\\\\n"""', (59, 23, 59, 26): '""""""', (59, 28, 59, 29): 's'}, {}), "('\\\\\\\\n', '', s)", False, 'import re\n'), ((66, 8, 66, 28), 're.sub', 're.sub', ({(66, 15, 66, 19): '"""\'"""', (66, 21, 66, 24): '""""""', (66, 26, 66, 27): 's'}, {}), '("\'", \'\', s)', False, 'import re\n'), ((67, 8, 67, 30), 're.sub', 're.sub', ({(67, 15, 67, 21): '"""“"""', (67, 23, 67, 26): '""""""', (67, 28, 67, 29): 's'}, {}), "('“', '', s)", False, 'import re\n'), ((68, 8, 68, 30), 're.sub', 're.sub', ({(68, 15, 68, 21): '"""”"""', (68, 23, 68, 26): '""""""', (68, 28, 68, 29): 's'}, {}), "('”', '', s)", False, 'import re\n'), ((69, 8, 69, 30), 're.sub', 're.sub', ({(69, 15, 69, 21): '"""[()]"""', (69, 23, 69, 26): '""""""', (69, 28, 69, 29): 's'}, {}), "('[()]', '', s)", False, 'import re\n'), ((70, 8, 70, 28), 're.sub', 're.sub', ({(70, 15, 70, 19): '"""\\""""', (70, 21, 70, 24): '""""""', (70, 26, 70, 27): 's'}, {}), '(\'"\', \'\', s)', False, 'import re\n'), ((73, 8, 73, 35), 're.sub', 're.sub', ({(73, 15, 73, 24): '"""([.-])+"""', (73, 26, 73, 31): '"""\\\\1"""', (73, 33, 73, 34): 's'}, {}), "('([.-])+', '\\\\1', s)", False, 'import re\n'), ((76, 8, 76, 58), 're.sub', 're.sub', ({(76, 15, 76, 41): '"""([^0-9])([.,!?])([^0-9])"""', (76, 43, 76, 54): '"""\\\\1 \\\\2 \\\\3"""', (76, 56, 76, 57): 's'}, {}), "('([^0-9])([.,!?])([^0-9])', '\\\\1 \\\\2 \\\\3', s)", False, 'import re\n'), ((82, 20, 98, 31), 're.compile', 're.compile', (), '', False, 'import re\n'), ((17, 23, 17, 39), 'json.load', 'json.load', ({(17, 33, 17, 38): 'input'}, {}), '(input)', False, 'import json\n'), ((34, 29, 34, 66), 'json.dumps', 'json.dumps', (), '', False, 'import json\n')]