Theory-based quiz grader using similarity
There is no need to include stop words in the dictionary if they are filtered out of the vectors anyway. Thus, tokenization and stop words removal can be considered as a single step. Secondly, the vectors can be initialized with the correct values immediately by taking advantage of collections.Counter
from collections import Counter
from typing import List
from nltk.corpus import stopwords
from nltk.cluster.util import cosine_distance
from nltk.tokenize import word_tokenize

def tokenize_without_stop_words(sentence:str) -&gt; List[str]:
    return [
        word
        for word in word_tokenize(sentence)
        if word not in stopwords.words('english')
    ]

def cosine_similarity(first_sentence: str, second_sentence: str) -&gt; float:
    first_words = tokenize_without_stop_words(first_sentence)
    second_words = tokenize_without_stop_words(second_sentence)

    dictionary = list(set(first_words+second_words))

    def encode(words: List[str]) -&gt; List[int]:
        word_counts = Counter(words)
        return [
            word_counts[word]
            for word in dictionary
        ]

    first_vector = encode(first_words)
    second_vector = encode(second_words)

    return 1 - cosine_distance(first_vector, second_vector)