import re
from typing import Union, List

import nltk
from bs4 import BeautifulSoup


class Normalizer:
    
    def __init__(self):
        self.lemmatizer = nltk.stem.WordNetLemmatizer()
    
    def normalize(self, x: Union[list, str]) -> List[str]:
        """
        Accepts text (possibly tokenized) and makes it suitable for machine processing
        """
        x = self._remove_stop_words(x)
        x = self._denoise(x)
        x = self._lemmatize(x)
        return x
    
    def _remove_stop_words(self, x: Union[list, str]) -> List[str]:
        """
        Removes stop words from text in english
        """
        if isinstance(x, str):
            x = x.split(' ')
        stop_words = set(nltk.corpus.stopwords.words('english')) 
        return [w for w in x if not w in stop_words]
    
    def _lemmatize(self, x: Union[list, str]) -> List[str]:
        """
        Removes endings,
        """
        if isinstance(x, list):
            x = ' '.join(x)
        x = self.lemmatizer.lemmatize(x)
        return x
    
    def _denoise(self, x: Union[list, str]) -> str:
        if isinstance(x, list):
            x = ' '.join(x)
            
        def strip_html(x):
            soup = BeautifulSoup(x, "html.parser")
            x = soup.get_text()
            return x

        def remove_between_square_brackets(x):
            x = re.sub('\[[^]]*\]', '', x)
            x = re.sub(r'http\S+', '', x)
            return x
        
        def remove_rating(x):
            return re.sub('\W\d/\d+\S*', '', x)
        
        x = x.lower()
        x = re.sub(',|\.|!|\?', '', x)
        x = strip_html(x)
        x = remove_between_square_brackets(x)
        x = remove_rating(x)
        return x
[((11, 26, 11, 55), 'nltk.stem.WordNetLemmatizer', 'nltk.stem.WordNetLemmatizer', ({}, {}), '()', False, 'import nltk\n'), ((58, 12, 58, 38), 're.sub', 're.sub', ({(58, 19, 58, 30): '""",|\\\\.|!|\\\\?"""', (58, 32, 58, 34): '""""""', (58, 36, 58, 37): 'x'}, {}), "(',|\\\\.|!|\\\\?', '', x)", False, 'import re\n'), ((28, 25, 28, 63), 'nltk.corpus.stopwords.words', 'nltk.corpus.stopwords.words', ({(28, 53, 28, 62): '"""english"""'}, {}), "('english')", False, 'import nltk\n'), ((45, 19, 45, 50), 'bs4.BeautifulSoup', 'BeautifulSoup', ({(45, 33, 45, 34): 'x', (45, 36, 45, 49): '"""html.parser"""'}, {}), "(x, 'html.parser')", False, 'from bs4 import BeautifulSoup\n'), ((50, 16, 50, 42), 're.sub', 're.sub', ({(50, 23, 50, 34): '"""\\\\[[^]]*\\\\]"""', (50, 36, 50, 38): '""""""', (50, 40, 50, 41): 'x'}, {}), "('\\\\[[^]]*\\\\]', '', x)", False, 'import re\n'), ((51, 16, 51, 41), 're.sub', 're.sub', ({(51, 23, 51, 33): '"""http\\\\S+"""', (51, 35, 51, 37): '""""""', (51, 39, 51, 40): 'x'}, {}), "('http\\\\S+', '', x)", False, 'import re\n'), ((55, 19, 55, 47), 're.sub', 're.sub', ({(55, 26, 55, 39): '"""\\\\W\\\\d/\\\\d+\\\\S*"""', (55, 41, 55, 43): '""""""', (55, 45, 55, 46): 'x'}, {}), "('\\\\W\\\\d/\\\\d+\\\\S*', '', x)", False, 'import re\n')]