Train a deep learning model in chunks/sequentially to avoid memory error How do I train/fit a model in chunks so as to escape the dreaded memory error?

def TFIDF(X_train, X_test, MAX_NB_WORDS=75000):
    vectorizer_x = TfidfVectorizer(max_features=MAX_NB_WORDS)
    X_train = vectorizer_x.fit_transform(X_train).toarray()
    X_test = vectorizer_x.transform(X_test).toarray()
    print("tf-idf with", str(np.array(X_train).shape[1]), "features")
    return (X_train, X_test)


# In[3]:


def Build_Model_DNN_Text(shape, nClasses, dropout=0.5):
    """
    buildModel_DNN_Tex(shape, nClasses,dropout)
    Build Deep neural networks Model for text classification
    Shape is input feature space
    nClasses is number of classes
    """
    model = Sequential()
    node = 512  # number of nodes
    nLayers = 4  # number of  hidden layer
    model.add(Dense(node, input_dim=shape, activation='relu'))
    model.add(Dropout(dropout))
    for i in range(0, nLayers):
        model.add(Dense(node, input_dim=node, activation='relu'))
        model.add(Dropout(dropout))
    model.add(Dense(nClasses, activation='softmax'))
    model.compile(loss='sparse_categorical_crossentropy',
                  optimizer='adam',
                  metrics=['accuracy'])
    return model


# In[18]:


df = pd.read_csv("ExtractedData.csv")

# In[19]:

df = df.dropna()
X_train, X_test, y_train, y_test = train_test_split(df['body'], df['user_id'],
                                                    test_size=0.3, random_state=42, shuffle=True)
X_train_tfidf, X_test_tfidf = TFIDF(X_train, X_test)
y_train_tfidf, y_test_tfidf = TFIDF(y_train,y_test)
output_nodes = len(list(set(df['user_id'])))
model_DNN = Build_Model_DNN_Text(X_train_tfidf.shape[1], output_nodes)
model_DNN.fit(X_train_tfidf, y_train_tfidf,
              validation_data=(X_test_tfidf, y_test_tfidf),
              epochs=10,
              batch_size=256,
              verbose=2)

predicted = model_DNN.predict(X_test_tfidf)
print(metrics.classification_report(y_test, predicted))


When I run the above, I keep getting the memory error: Unable to allocate 37.9 GiB for an array with shape (67912, 75000) and data type float64. I know I can send the data in chunks pd.read_csv( , chunksize= 5000), but what I don't know is how do I implement it here. My data has 98000 rows and 4 columns. Thank you.
Pandas is not memory efficient for data loading.

It appears you are using Keras / TensorFlow package. It would be better to use that package's data loading for csv which will better manage memory.