week6/questions/questions.py
def tokenize(document):
    """
    Given a document (represented as a string), return a list of all of the
    words in that document, in order.

    Process document by converting all words to lowercase, and removing any
    punctuation or English stopwords.
    """
    words = nltk.word_tokenize(document.lower())                # Tokenize all words in the document and lowercase

    stopwords = set(nltk.corpus.stopwords.words('english'))     # Initialize a set with all stop words
    punctuation = set(string.punctuation)                       # Initialize a set with all punctuation
    to_be_removed = set()                                       # A set to store all words that need to be removed

    n = len(words)

    # Go over all words in the list and add those than need to be removed to the set
    for i in range(0, n):
        word = words[i]

        # If word is a stop word or punctuation, it should be removed
        if word in stopwords or word in punctuation:
            to_be_removed.add(word)

    # return a list of words that are not meant to be removed
    return [word for word in words if word not in to_be_removed]

    Given a document (represented as a string), return a list of all of the
    words in that document, in order.

    Process document by converting all words to lowercase, and removing any
    punctuation or English stopwords.
    
Given a document (represented as a string), return a list of all of the
words in that document, in order.

Process document by converting all words to lowercase, and removing any
punctuation or English stopwords.
Given a document ( represented as a string ) return a list of all of the words in that document in order . Process document by converting all words to lowercase and removing any punctuation or English stopwords .
def tokenize(document):
    words = nltk.word_tokenize(document.lower())                
    stopwords = set(nltk.corpus.stopwords.words('english'))     
    punctuation = set(string.punctuation)                       
    to_be_removed = set()                                       
    n = len(words)
    for i in range(0, n):
        word = words[i]
        if word in stopwords or word in punctuation:
            to_be_removed.add(word)
    return [word for word in words if word not in to_be_removed]
def tokenize ( document ) : words = nltk . word_tokenize ( document . lower ( ) ) stopwords = set ( nltk . corpus . stopwords . words ( 'english' ) ) punctuation = set ( string . punctuation ) to_be_removed = set ( ) n = len ( words ) for i in range ( 0 , n ) : word = words [ i ] if word in stopwords or word in punctuation : to_be_removed . add ( word ) return [ word for word in words if word not in to_be_removed ]
Given a document (represented as a string), return a list of all of the
words in that document, in order.
Given a document ( represented as a string ) return a list of all of the words in that document in order .
"""
    Given a document (represented as a string), return a list of all of the
    words in that document, in order.

    Process document by converting all words to lowercase, and removing any
    punctuation or English stopwords.
    """ # Tokenize all words in the document and lowercase # Initialize a set with all stop words # Initialize a set with all punctuation # A set to store all words that need to be removed # Go over all words in the list and add those than need to be removed to the set # If word is a stop word or punctuation, it should be removed # return a list of words that are not meant to be removed
document