def extract_graphs_and_lemmas_from_tweets(tweet_generator):
    """
    Given a tweet python generator, we encode the information into mention and retweet graphs and a lemma matrix.

    We assume that the tweets are given in increasing timestamp.

    Inputs:  - tweet_generator: A python generator of tweets in python dictionary (json) format.

    Outputs: - mention_graph: The mention graph as a SciPy sparse matrix.
             - retweet_graph: The retweet graph as a SciPy sparse matrix.
             - user_lemma_matrix: The user lemma vector representation matrix as a SciPy sparse matrix.
             - tweet_id_set: A python set containing the Twitter ids for all the dataset tweets.
             - user_id_set: A python set containing the Twitter ids for all the dataset users.
             - lemma_to_attribute: A map from lemmas to numbers in python dictionary format.
    """
    ####################################################################################################################
    # Prepare for iterating over tweets.
    ####################################################################################################################
    # These are initialized as lists for incremental extension.
    tweet_id_set = set()
    user_id_set = list()

    add_tweet_id = tweet_id_set.add
    append_user_id = user_id_set.append

    # Initialize sparse matrix arrays.
    mention_graph_row = list()
    mention_graph_col = list()

    retweet_graph_row = list()
    retweet_graph_col = list()

    user_lemma_matrix_row = list()
    user_lemma_matrix_col = list()
    user_lemma_matrix_data = list()

    append_mention_graph_row = mention_graph_row.append
    append_mention_graph_col = mention_graph_col.append

    append_retweet_graph_row = retweet_graph_row.append
    append_retweet_graph_col = retweet_graph_col.append

    extend_user_lemma_matrix_row = user_lemma_matrix_row.extend
    extend_user_lemma_matrix_col = user_lemma_matrix_col.extend
    extend_user_lemma_matrix_data = user_lemma_matrix_data.extend

    # Initialize dictionaries.
    id_to_node = dict()
    id_to_name = dict()
    id_to_username = dict()
    id_to_listedcount = dict()
    lemma_to_attribute = dict()

    sent_tokenize, _treebank_word_tokenize = get_tokenizer()
    # tagger = HunposTagger('hunpos-1.0-linux/english.model', 'hunpos-1.0-linux/hunpos-tag')
    # tagger = PerceptronTagger()
    tagger = get_braupt_tagger()
    lemmatizer, lemmatize = get_lemmatizer("wordnet")
    stopset = get_stopset()
    first_cap_re, all_cap_re = get_camel_case_regexes()
    digits_punctuation_whitespace_re = get_digits_punctuation_whitespace_regex()
    pos_set = get_pos_set()

    ####################################################################################################################
    # Iterate over tweets.
    ####################################################################################################################
    counter = 0
    for tweet in tweet_generator:
        # print(tweet)
        # Increment tweet counter.
        counter += 1
        # if counter % 10000 == 0:
        #     print(counter)
        # print(counter)

        # Extract base tweet's values.
        try:
            tweet_id = tweet["id"]
            user_id = tweet["user"]["id"]
            user_name = tweet["user"]["name"]
            user_screen_name = tweet["user"]["screen_name"]

            listed_count_raw = tweet["user"]["listed_count"]

            tweet_text = tweet["text"]

            tweet_in_reply_to_user_id = tweet["in_reply_to_user_id"]
            tweet_in_reply_to_screen_name = tweet["in_reply_to_screen_name"]
            tweet_entities_user_mentions = tweet["entities"]["user_mentions"]

            if "retweeted_status" not in tweet.keys():
                user_mention_id_list = list()
                user_mention_screen_name_list = list()
                for user_mention in tweet_entities_user_mentions:
                    user_mention_id_list.append(user_mention["id"])
                    user_mention_screen_name_list.append(user_mention["screen_name"])
            else:
                # Extract base tweet's values.
                original_tweet = tweet["retweeted_status"]

                original_tweet_id = original_tweet["id"]
                original_tweet_user_id = original_tweet["user"]["id"]
                original_tweet_user_name = original_tweet["user"]["name"]
                original_tweet_user_screen_name = original_tweet["user"]["screen_name"]

                listed_count_raw = original_tweet["user"]["listed_count"]

                original_tweet_text = original_tweet["text"]

                original_tweet_in_reply_to_user_id = original_tweet["in_reply_to_user_id"]
                original_tweet_in_reply_to_screen_name = original_tweet["in_reply_to_screen_name"]
                original_tweet_entities_user_mentions = original_tweet["entities"]["user_mentions"]

                original_tweet_user_mention_id_list = list()
                original_tweet_user_mention_screen_name_list = list()
                for user_mention in original_tweet_entities_user_mentions:
                    original_tweet_user_mention_id_list.append(user_mention["id"])
                    original_tweet_user_mention_screen_name_list.append(user_mention["screen_name"])
        except KeyError:
            continue
        # Map users to distinct integer numbers.
        graph_size = len(id_to_node)
        source_node = id_to_node.setdefault(user_id, graph_size)

        if listed_count_raw is None:
            id_to_listedcount[user_id] = 0
        else:
            id_to_listedcount[user_id] = int(listed_count_raw)

        # Update sets, lists and dictionaries.
        add_tweet_id(tweet_id)
        id_to_name[user_id] = user_screen_name
        id_to_username[user_id] = user_name
        append_user_id(user_id)

        ################################################################################################################
        # We are dealing with an original tweet.
        ################################################################################################################
        if "retweeted_status" not in tweet.keys():
            ############################################################################################################
            # Update user-lemma frequency matrix.
            ############################################################################################################
            # Extract lemmas from the text.
            tweet_lemmas, lemma_to_keywordbag = clean_document(tweet_text, sent_tokenize, _treebank_word_tokenize,
                                                               tagger, lemmatizer, lemmatize, stopset,
                                                               first_cap_re, all_cap_re, digits_punctuation_whitespace_re,
                                                               pos_set)

            number_of_lemmas = len(tweet_lemmas)

            # Update the user-lemma frequency matrix one-by-one.
            attribute_list = list()
            append_attribute = attribute_list.append
            for lemma in tweet_lemmas:
                # Map lemmas to distinct integer numbers.
                vocabulary_size = len(lemma_to_attribute)
                attribute = lemma_to_attribute.setdefault(lemma, vocabulary_size)
                append_attribute(attribute)

            # Add values to the sparse matrix arrays.
            extend_user_lemma_matrix_row(number_of_lemmas*[source_node])
            extend_user_lemma_matrix_col(attribute_list)
            extend_user_lemma_matrix_data(number_of_lemmas*[1.0])

            ############################################################################################################
            # Update mention matrix.
            ############################################################################################################
            # Get mentioned user ids.
            mentioned_user_id_set = list()
            if tweet_in_reply_to_user_id is not None:
                mentioned_user_id_set.append(tweet_in_reply_to_user_id)

                id_to_name[tweet_in_reply_to_user_id] = tweet_in_reply_to_screen_name
            for user_mention, mentioned_user_id, mentioned_user_screen_name in zip(tweet_entities_user_mentions,
                                                                                   user_mention_id_list,
                                                                                   user_mention_screen_name_list):
                mentioned_user_id_set.append(mentioned_user_id)

                id_to_name[mentioned_user_id] = mentioned_user_screen_name

            # We remove duplicates.
            mentioned_user_id_set = set(mentioned_user_id_set)

            # Update the mention graph one-by-one.
            for mentioned_user_id in mentioned_user_id_set:
                # Map users to distinct integer numbers.
                graph_size = len(id_to_node)
                mention_target_node = id_to_node.setdefault(mentioned_user_id, graph_size)

                append_user_id(mentioned_user_id)

                # Add values to the sparse matrix arrays.
                append_mention_graph_row(source_node)
                append_mention_graph_col(mention_target_node)

        ################################################################################################################
        # We are dealing with a retweet.
        ################################################################################################################
        else:
            # Map users to distinct integer numbers.
            graph_size = len(id_to_node)
            original_tweet_node = id_to_node.setdefault(original_tweet_user_id, graph_size)

            if listed_count_raw is None:
                id_to_listedcount[user_id] = 0
            else:
                id_to_listedcount[user_id] = int(listed_count_raw)

            # Update retweet graph.
            append_retweet_graph_row(source_node)
            append_retweet_graph_col(original_tweet_node)

            # Extract lemmas from the text.
            tweet_lemmas, lemma_to_keywordbag = clean_document(original_tweet_text, sent_tokenize, _treebank_word_tokenize,
                                                               tagger, lemmatizer, lemmatize, stopset,
                                                               first_cap_re, all_cap_re, digits_punctuation_whitespace_re,
                                                               pos_set)

            number_of_lemmas = len(tweet_lemmas)

            # Update the user-lemma frequency matrix one-by-one.
            attribute_list = list()
            append_attribute = attribute_list.append
            for lemma in tweet_lemmas:
                # Map lemmas to distinct integer numbers.
                vocabulary_size = len(lemma_to_attribute)
                attribute = lemma_to_attribute.setdefault(lemma, vocabulary_size)
                append_attribute(attribute)

            # Add values to the sparse matrix arrays.
            extend_user_lemma_matrix_row(number_of_lemmas*[source_node])
            extend_user_lemma_matrix_col(attribute_list)
            extend_user_lemma_matrix_data(number_of_lemmas*[1.0])

            # Get mentioned user ids.
            mentioned_user_id_set = list()
            if original_tweet_in_reply_to_user_id is not None:
                mentioned_user_id_set.append(original_tweet_in_reply_to_user_id)

                id_to_name[original_tweet_in_reply_to_user_id] = original_tweet_in_reply_to_screen_name
            for user_mention, mentioned_user_id, mentioned_user_screen_name in zip(original_tweet_entities_user_mentions,
                                                                                   original_tweet_user_mention_id_list,
                                                                                   original_tweet_user_mention_screen_name_list):
                mentioned_user_id_set.append(mentioned_user_id)

                id_to_name[mentioned_user_id] = mentioned_user_screen_name

            # We remove duplicates.
            mentioned_user_id_set = set(mentioned_user_id_set)

            # Get mentioned user ids.
            retweet_mentioned_user_id_set = list()
            if original_tweet_in_reply_to_user_id is not None:
                retweet_mentioned_user_id_set.append(original_tweet_in_reply_to_user_id)

                id_to_name[original_tweet_in_reply_to_user_id] = original_tweet_in_reply_to_screen_name
            for user_mention, mentioned_user_id, mentioned_user_screen_name in zip(original_tweet_entities_user_mentions,
                                                                                   original_tweet_user_mention_id_list,
                                                                                   original_tweet_user_mention_screen_name_list):
                retweet_mentioned_user_id_set.append(mentioned_user_id)

                id_to_name[mentioned_user_id] = mentioned_user_screen_name

            # We remove duplicates.
            retweet_mentioned_user_id_set = set(retweet_mentioned_user_id_set)

            mentioned_user_id_set.update(retweet_mentioned_user_id_set)

            # Update the mention graph one-by-one.
            for mentioned_user_id in mentioned_user_id_set:
                # Map users to distinct integer numbers.
                graph_size = len(id_to_node)
                mention_target_node = id_to_node.setdefault(mentioned_user_id, graph_size)

                append_user_id(mentioned_user_id)

                # Add values to the sparse matrix arrays.
                append_mention_graph_row(source_node)
                append_mention_graph_col(mention_target_node)

            # This is the first time we deal with this tweet.
            if original_tweet_id not in tweet_id_set:
                # Update sets, lists and dictionaries.
                add_tweet_id(original_tweet_id)
                id_to_name[original_tweet_user_id] = original_tweet_user_screen_name
                id_to_username[original_tweet_user_id] = original_tweet_user_name
                append_user_id(original_tweet_user_id)

                ########################################################################################################
                # Update user-lemma frequency matrix.
                ########################################################################################################
                # Update the user-lemma frequency matrix one-by-one.
                attribute_list = list()
                append_attribute = attribute_list.append
                for lemma in tweet_lemmas:
                    # Map lemmas to distinct integer numbers.
                    vocabulary_size = len(lemma_to_attribute)
                    attribute = lemma_to_attribute.setdefault(lemma, vocabulary_size)
                    append_attribute(attribute)

                # Add values to the sparse matrix arrays.
                extend_user_lemma_matrix_row(number_of_lemmas*[source_node])
                extend_user_lemma_matrix_col(attribute_list)
                extend_user_lemma_matrix_data(number_of_lemmas*[1.0])

                ########################################################################################################
                # Update mention matrix.
                ########################################################################################################
                # Update the mention graph one-by-one.
                for mentioned_user_id in retweet_mentioned_user_id_set:
                    # Map users to distinct integer numbers.
                    graph_size = len(id_to_node)
                    mention_target_node = id_to_node.setdefault(mentioned_user_id, graph_size)

                    append_user_id(mentioned_user_id)

                    # Add values to the sparse matrix arrays.
                    append_mention_graph_row(original_tweet_node)
                    append_mention_graph_col(mention_target_node)
            else:
                pass

    ####################################################################################################################
    # Final steps of preprocessing tweets.
    ####################################################################################################################
    # Discard any duplicates.
    user_id_set = set(user_id_set)
    number_of_users = len(user_id_set)
    # min_number_of_users = max(user_id_set) + 1

    # Form mention graph adjacency matrix.
    mention_graph_row = np.array(mention_graph_row, dtype=np.int64)
    mention_graph_col = np.array(mention_graph_col, dtype=np.int64)
    mention_graph_data = np.ones_like(mention_graph_row, dtype=np.float64)

    mention_graph = spsp.coo_matrix((mention_graph_data, (mention_graph_row, mention_graph_col)),
                                    shape=(number_of_users, number_of_users))
    mention_graph = spsp.coo_matrix(spsp.csr_matrix(mention_graph))

    # Form retweet graph adjacency matrix.
    retweet_graph_row = np.array(retweet_graph_row, dtype=np.int64)
    retweet_graph_col = np.array(retweet_graph_col, dtype=np.int64)
    retweet_graph_data = np.ones_like(retweet_graph_row, dtype=np.float64)

    retweet_graph = spsp.coo_matrix((retweet_graph_data, (retweet_graph_row, retweet_graph_col)),
                                    shape=(number_of_users, number_of_users))
    retweet_graph = spsp.coo_matrix(spsp.csr_matrix(retweet_graph))

    # Form user-lemma matrix.
    number_of_lemmas = len(lemma_to_attribute)

    user_lemma_matrix_row = np.array(user_lemma_matrix_row, dtype=np.int64)
    user_lemma_matrix_col = np.array(user_lemma_matrix_col, dtype=np.int64)
    user_lemma_matrix_data = np.array(user_lemma_matrix_data, dtype=np.float64)

    user_lemma_matrix = spsp.coo_matrix((user_lemma_matrix_data, (user_lemma_matrix_row, user_lemma_matrix_col)),
                                        shape=(number_of_users, number_of_lemmas))
    user_lemma_matrix = spsp.coo_matrix(spsp.csr_matrix(user_lemma_matrix))

    node_to_id = dict(zip(id_to_node.values(), id_to_node.keys()))

    # tagger.close()

    return mention_graph, retweet_graph, user_lemma_matrix, tweet_id_set, user_id_set, node_to_id, lemma_to_attribute, id_to_name, id_to_username, id_to_listedcount