import re
from typing import Union, List

import nltk
from bs4 import BeautifulSoup


class Normalizer:
    
    def __init__(self):
        self.lemmatizer = nltk.stem.WordNetLemmatizer()
    
    def normalize(self, x: Union[list, str]) -> List[str]:
        """
        Accepts text (possibly tokenized) and makes it suitable for machine processing
        """
        x = self._remove_stop_words(x)
        x = self._denoise(x)
        x = self._lemmatize(x)
        return x
    
    def _remove_stop_words(self, x: Union[list, str]) -> List[str]:
        """
        Removes stop words from text in english
        """
        if isinstance(x, str):
            x = x.split(' ')
        stop_words = set(nltk.corpus.stopwords.words('english')) 
        return [w for w in x if not w in stop_words]
    
    def _lemmatize(self, x: Union[list, str]) -> List[str]:
        """
        Removes endings,
        """
        if isinstance(x, list):
            x = ' '.join(x)
        x = self.lemmatizer.lemmatize(x)
        return x
    
    def _denoise(self, x: Union[list, str]) -> str:
        if isinstance(x, list):
            x = ' '.join(x)
            
        def strip_html(x):
            soup = BeautifulSoup(x, "html.parser")
            x = soup.get_text()
            return x

        def remove_between_square_brackets(x):
            x = re.sub('\[[^]]*\]', '', x)
            x = re.sub(r'http\S+', '', x)
            return x
        
        def remove_rating(x):
            return re.sub('\W\d/\d+\S*', '', x)
        
        x = x.lower()
        x = re.sub(',|\.|!|\?', '', x)
        x = strip_html(x)
        x = remove_between_square_brackets(x)
        x = remove_rating(x)
        return x
[((159, 188), 'nltk.stem.WordNetLemmatizer', 'nltk.stem.WordNetLemmatizer', ([], {}), '()\n', (186, 188), False, 'import nltk\n'), ((1581, 1609), 're.sub', 're.sub', (['""",|\\\\.|!|\\\\?"""', '""""""', 'x'], {}), "(',|\\\\.|!|\\\\?', '', x)\n", (1587, 1609), False, 'import re\n'), ((710, 748), 'nltk.corpus.stopwords.words', 'nltk.corpus.stopwords.words', (['"""english"""'], {}), "('english')\n", (737, 748), False, 'import nltk\n'), ((1212, 1243), 'bs4.BeautifulSoup', 'BeautifulSoup', (['x', '"""html.parser"""'], {}), "(x, 'html.parser')\n", (1225, 1243), False, 'from bs4 import BeautifulSoup\n'), ((1361, 1389), 're.sub', 're.sub', (['"""\\\\[[^]]*\\\\]"""', '""""""', 'x'], {}), "('\\\\[[^]]*\\\\]', '', x)\n", (1367, 1389), False, 'import re\n'), ((1404, 1429), 're.sub', 're.sub', (['"""http\\\\S+"""', '""""""', 'x'], {}), "('http\\\\S+', '', x)\n", (1410, 1429), False, 'import re\n'), ((1509, 1541), 're.sub', 're.sub', (['"""\\\\W\\\\d/\\\\d+\\\\S*"""', '""""""', 'x'], {}), "('\\\\W\\\\d/\\\\d+\\\\S*', '', x)\n", (1515, 1541), False, 'import re\n')]