"""
If you are Chinese, you can check out my [blog](https://blog.csdn.net/qq_33758867/article/details/90047240).
"""
"""
In digit recognizer, I initially learned to construct my own CNN model step by step, and finally used multi-model voting to get a good score. This article has a lot of training, and my GPU is not very good, 1050ti, which takes a lot of time. It is recommended that no GPU students do not train, you can take a look, then use the final model and the weight file I provided.
"""
"""
**Module Imports**
"""
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn import preprocessing
from keras import layers
from keras.layers import Input, Dense, Activation, ZeroPadding2D, BatchNormalization, Flatten, Conv2D, Add
from keras.layers import AveragePooling2D, MaxPooling2D, Dropout,GlobalMaxPooling2D, GlobalAveragePooling2D
from keras.models import Model, Sequential
from keras.callbacks import LearningRateScheduler, ReduceLROnPlateau
from keras.initializers import glorot_uniform
from keras.optimizers import RMSprop
import keras.backend as K
K.set_image_data_format("channels_last")
import time
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OneHotEncoder
from keras.preprocessing.image import ImageDataGenerator
"""
**Load Data**
"""
train = pd.read_csv("../input/train.csv")
test = pd.read_csv("../input/test.csv")
sample_submission = pd.read_csv("../input/sample_submission.csv")
"""
**Data preprocessing**
"""
label = np.array(train.label)
data = np.array(train.drop(["label"], axis=1))
 
# Normalization and one hot
X = data.reshape((42000, 28, 28, 1))
X = X / 255
enc = OneHotEncoder()
Y = enc.fit_transform(label.reshape((42000, 1))).toarray()
train_x, val_x, train_y, val_y = train_test_split(X, Y, test_size=0.1, random_state=0)
 
# Normalizetion
test_data = np.array(test)
test_data = test_data.reshape((28000, 28, 28, 1))
test_data = test_data / 255
 
# Take a look at the data at will
index = 25351
plt.imshow(data.reshape((42000, 28, 28))[index])
label[index]
"""
**Data augmentation**
"""
"""
Perform rotation on the picture, move up and down left and right, and other operations to increase the amount of data. **Data augmentation is very effective**. Parameters such as **rotation_range, zoom_range, width_shift_range, and height_shift_range can be modified**. The following parameters are the version used by most people, but you can also explore parameters that are more effective for your own model.
"""
datagen = ImageDataGenerator(featurewise_center=False, samplewise_center=False, featurewise_std_normalization=False, samplewise_std_normalization=False, zca_whitening=False, rotation_range=10, zoom_range=0.1, width_shift_range=0.1, height_shift_range=0.1, horizontal_flip=False, vertical_flip=False)
datagen.fit(train_x)
"""
**Build your own CNN step by step**. For the convenience of commit, all steps are trained for only one epoch. To achieve good results, you should train at least 20 epochs.
"""
"""
*First select the first layer filters parameter*
"""
model = [0] * 8
for j in range(0, 8):
    model[j] = Sequential()
    model[j].add(Conv2D(2 ** (j + 1), kernel_size=2, activation='relu', input_shape=(28,28,1)))
    model[j].add(Flatten())
    model[j].add(Dense(256, activation="relu"))
    model[j].add(Dense(10, activation='softmax'))
    model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
    print("---------------------------------------------------------------------------------------------")
    print(str(2 ** (j + 1)) + "个filters:")
    start = time.time()
    model[j].fit_generator(datagen.flow(train_x, train_y, batch_size=64), validation_data=(val_x, val_y), epochs=1, steps_per_epoch= len(train_x) // 64)
    end = time.time()
    print(str(end - start) + "秒")
    print("---------------------------------------------------------------------------------------------")
"""
Observe the cross-validation score of the training output and select the better parameters. My result was 32 or 64, and **I chose 32** to continue building.
"""
"""
*Select the first layer kernel_size parameter*
"""
model = [0] * 6
for j in range(6):
    model[j] = Sequential()
    model[j].add(Conv2D(32, kernel_size=2 + j, activation='relu', input_shape=(28,28,1)))
    model[j].add(Flatten())
    model[j].add(Dense(256, activation="relu"))
    model[j].add(Dense(10, activation='softmax'))
    model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
    print("---------------------------------------------------------------------------------------------")
    print("过滤器大小：" + str(2 + j))
    start = time.time()
    model[j].fit_generator(datagen.flow(train_x, train_y, batch_size=64), validation_data=(val_x, val_y), epochs=1, steps_per_epoch= len(train_x) // 64)
    end = time.time()
    print(str(end - start) + "秒")
    print("---------------------------------------------------------------------------------------------")
"""
3, 4, 5, 6, 7 all look good, **I chose 5** to continue to build.
"""
"""
*Padding parameter selection*
"""
model = Sequential()
model.add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
model.add(Flatten())
model.add(Dense(256, activation="relu"))
model.add(Dense(10, activation='softmax'))
model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
print("---------------------------------------------------------------------------------------------")
print("same：")
start = time.time()
model.fit_generator(datagen.flow(train_x, train_y, batch_size=64), validation_data=(val_x, val_y), epochs=20, steps_per_epoch= len(train_x) // 64)
end = time.time()
print(str(end - start) + "秒")
print("---------------------------------------------------------------------------------------------")
 
model = Sequential()
model.add(Conv2D(32, kernel_size=5, padding="valid", activation='relu', input_shape=(28,28,1)))
model.add(Flatten())
model.add(Dense(256, activation="relu"))
model.add(Dense(10, activation='softmax'))
model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
print("---------------------------------------------------------------------------------------------")
print("same：")
start = time.time()
model.fit_generator(datagen.flow(train_x, train_y, batch_size=64), validation_data=(val_x, val_y), epochs=1, steps_per_epoch= len(train_x) // 64)
end = time.time()
print(str(end - start) + "秒")
print("---------------------------------------------------------------------------------------------")
"""
**Same seems a little better**, but it doesn't make much difference.
"""
"""
*Select the number of convolution layers*
"""
model = [0] * 5
for j in range(5):
    model[j] = Sequential()
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    
    if j > 0:
        model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    if j > 1:
        model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    if j > 2:
        model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    if j > 3:
        model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    
    model[j].add(Flatten())
    model[j].add(Dense(256, activation="relu"))
    model[j].add(Dense(10, activation='softmax'))
    model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
    print("---------------------------------------------------------------------------------------------")
    print("卷积层层数：" + str(1 + j))
    start = time.time()
    model[j].fit_generator(datagen.flow(train_x, train_y, batch_size=64), validation_data=(val_x, val_y), epochs=1, steps_per_epoch= len(train_x) // 64)
    end = time.time()
    print(str(end - start) + "秒")
    print("---------------------------------------------------------------------------------------------")
"""
3, 4, 5 are optional.
"""
"""
*Select pooling layer pool_size parameter*
"""
model = [0] * 5
for j in range(5):
    model[j] = Sequential()
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=j+2))
    model[j].add(Flatten())
    model[j].add(Dense(256, activation="relu"))
    model[j].add(Dense(10, activation='softmax'))
    model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
    print("---------------------------------------------------------------------------------------------")
    print("池化大小：" + str(2 + j))
    start = time.time()
    model[j].fit_generator(datagen.flow(train_x, train_y, batch_size=64), validation_data=(val_x, val_y), epochs=1, steps_per_epoch= len(train_x) // 64)
    end = time.time()
    print(str(end - start) + "秒")
    print("---------------------------------------------------------------------------------------------")
"""
2, 3, 4 all look good.
"""
"""
Decided to use **5 layers of convolution, pool_size=3** continues to build.
"""
"""
***Choose how to add pool layer***, starting with the last convolution layer, and adding a pool layer up to each convolution layer. The result of training observation is that the pool layer is added after the last convolution layer, or the pool layer is added after the countdown first and second convolution layer. **I chose to add the pooling layer to the bottom one or two layers, that is, the last two layers**.
"""
model = Sequential()
model.add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
model.add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
model.add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
model.add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
model.add(MaxPooling2D(pool_size=3, padding="same"))
model.add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
model.add(MaxPooling2D(pool_size=3))
 
model.add(Flatten())
model.add(Dense(256, activation="relu"))
model.add(Dense(10, activation='softmax'))
model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
print("---------------------------------------------------------------------------------------------")
print("最后两层池化：")
start = time.time()
model.fit_generator(datagen.flow(train_x, train_y, batch_size=64), validation_data=(val_x, val_y), epochs=1, steps_per_epoch= len(train_x) // 64)
end = time.time()
print(str(end - start) + "秒")
print("---------------------------------------------------------------------------------------------")
"""
Based on the model that has been built, that is, the five layers of convolution layer plus the last two layers are pooled into a module, and *the number of modules required is selected*.
"""
model = [0] * 2
for j in range(2):
    model[j] = Sequential()
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    
    if j > 0:
        model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
        model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
        model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
        model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
        model[j].add(MaxPooling2D(pool_size=3, padding="same"))
        model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
        model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    
    model[j].add(Flatten())
    model[j].add(Dense(256, activation="relu"))
    model[j].add(Dense(10, activation='softmax'))
    model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
    print("---------------------------------------------------------------------------------------------")
    print("模块层数：" + str(1 + j))
    start = time.time()
    model[j].fit_generator(datagen.flow(train_x, train_y, batch_size=64), validation_data=(val_x, val_y), epochs=1, steps_per_epoch= len(train_x) // 64)
    end = time.time()
    print(str(end - start) + "秒")
    print("---------------------------------------------------------------------------------------------")
"""
A module count of **2** appears to be valid.
"""
"""
*Selection of filter parameters for convolution layer of New Module*
"""
model = [0] * 2
for j in range(2):
    model[j] = Sequential()
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    
    model[j].add(Conv2D(32 * 2 ** (j + 1), kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32 * 2 ** (j + 1), kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32 * 2 ** (j + 1), kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32 * 2 ** (j + 1), kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    model[j].add(Conv2D(32 * 2 ** j, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    
    model[j].add(Flatten())
    model[j].add(Dense(256, activation="relu"))
    model[j].add(Dense(10, activation='softmax'))
    model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
    print("---------------------------------------------------------------------------------------------")
    print("新模块filters数：" + str(32 * 2 ** (j + 1)))
    start = time.time()
    model[j].fit_generator(datagen.flow(train_x, train_y, batch_size=64), validation_data=(val_x, val_y), epochs=1, steps_per_epoch= len(train_x) // 64)
    end = time.time()
    print(str(end - start) + "秒")
    print("---------------------------------------------------------------------------------------------")
"""
**Choose 64**
"""
"""
*Selection of kernel_size parameters for convolution layer of New Module*
"""
model = [0] * 6
for j in range(6):
    model[j] = Sequential()
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    
    model[j].add(Conv2D(64, kernel_size=2 + j, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(64, kernel_size=2 + j, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(64, kernel_size=2 + j, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(64, kernel_size=2 + j, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    model[j].add(Conv2D(64, kernel_size=2 + j, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    
    model[j].add(Flatten())
    model[j].add(Dense(256, activation="relu"))
    model[j].add(Dense(10, activation='softmax'))
    model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
    print("---------------------------------------------------------------------------------------------")
    print("新模块过滤器大小：" + str(2 + j))
    start = time.time()
    model[j].fit_generator(datagen.flow(train_x, train_y, batch_size=64), validation_data=(val_x, val_y), epochs=1, steps_per_epoch= len(train_x) // 64)
    end = time.time()
    print(str(end - start) + "秒")
    print("---------------------------------------------------------------------------------------------")
"""
It feels like 3, 4, 5, 6 are all the same, **continue to use 5**
"""
"""
*Pool_size Parameter selection of New Module pool layer*
"""
model = [0] * 5
for j in range(5):
    model[j] = Sequential()
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    
    model[j].add(Conv2D(64, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(64, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(64, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(64, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=2 + j, padding="same"))
    model[j].add(Conv2D(64, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=2 + j, padding="same"))
    
    model[j].add(Flatten())
    model[j].add(Dense(256, activation="relu"))
    model[j].add(Dense(10, activation='softmax'))
    model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
    print("---------------------------------------------------------------------------------------------")
    print("新模块池化大小：" + str(2 + j))
    start = time.time()
    model[j].fit_generator(datagen.flow(train_x, train_y, batch_size=64), validation_data=(val_x, val_y), epochs=1, steps_per_epoch= len(train_x) // 64)
    end = time.time()
    print(str(end - start) + "秒")
    print("---------------------------------------------------------------------------------------------")
"""
Still use **3**.
"""
"""
With the two convolution pooling modules of the current model and the final fully connected module, a total of three modules are *selected to add dropout*. Each module is added separately, three modules are added, and any two modules are added in a total of 7 ways.
"""
model = [0] * 7
for j in range(7):
    model[j] = Sequential()
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    if j == 0 or j == 3 or j == 4 or j == 6:
        model[j].add(Dropout(0.1))
    
    model[j].add(Conv2D(64, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(64, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(64, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(64, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    model[j].add(Conv2D(64, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    if j == 1 or j == 3 or j == 5 or j == 6:
        model[j].add(Dropout(0.1))
    
    model[j].add(Flatten())
    model[j].add(Dense(256, activation="relu"))
    
    if j == 2 or j == 4 or j == 5 or j == 6:
        model[j].add(Dropout(0.1))
    
    model[j].add(Dense(10, activation='softmax'))
    model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
    print("---------------------------------------------------------------------------------------------")
    print("添加dropout的方式：" + str(j))
    start = time.time()
    model[j].fit_generator(datagen.flow(train_x, train_y, batch_size=64), validation_data=(val_x, val_y), epochs=1, steps_per_epoch= len(train_x) // 64)
    end = time.time()
    print(str(end - start) + "秒")
    print("---------------------------------------------------------------------------------------------")
"""
Choose the fifth way to add, that is, **add droupout after the second and third modules**
"""
"""
*Select the parameters of the first dropout*, dropout parameter selection training using the loop may encounter some problems, you can manually modify the parameter values training observation.
"""
model = [0] * 7
for j in range(7):
    model[j] = Sequential()
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    
    model[j].add(Conv2D(64, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(64, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(64, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(64, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    model[j].add(Conv2D(64, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    
    model[j].add(Dropout(0.1 * (j + 1)))
    
    model[j].add(Flatten())
    model[j].add(Dense(256, activation="relu"))
    
    model[j].add(Dropout(0.1))
    
    model[j].add(Dense(10, activation='softmax'))
    model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
    print("---------------------------------------------------------------------------------------------")
    print("第一个dropout的参数：" + str(0.1 * (j + 1)))
    start = time.time()
    model[j].fit_generator(datagen.flow(train_x, train_y, batch_size=64), validation_data=(val_x, val_y), epochs=1, steps_per_epoch= len(train_x) // 64)
    end = time.time()
    print(str(end - start) + "秒")
    print("---------------------------------------------------------------------------------------------")
"""
0.1, 0.4, 0.5, 0.6, 0.7 all seem good, **choose 0.4**
"""
"""
*Select the parameters for the second dropout*
"""
model = [0] * 6
for j in range(6):
    model[j] = Sequential()
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    model[j].add(Conv2D(32, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    
    model[j].add(Conv2D(64, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(64, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(64, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(Conv2D(64, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    model[j].add(Conv2D(64, kernel_size=5, padding="same", activation='relu', input_shape=(28,28,1)))
    model[j].add(MaxPooling2D(pool_size=3, padding="same"))
    
    model[j].add(Dropout(0.4))
    
    model[j].add(Flatten())
    model[j].add(Dense(256, activation="relu"))
    
    model[j].add(Dropout(0.2 + j * 0.1))
    
    model[j].add(Dense(10, activation='softmax'))
    model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
    print("---------------------------------------------------------------------------------------------")
    print("第二个dropout的参数：" + str(0.1 * (j + 2)))
    start = time.time()
    model[j].fit_generator(datagen.flow(train_x, train_y, batch_size=64), validation_data=(val_x, val_y), epochs=1, steps_per_epoch= len(train_x) // 64)
    end = time.time()
    print(str(end - start) + "秒")
    print("---------------------------------------------------------------------------------------------")
"""
0.2, 0.5 is better, **choose 0.5**
"""
"""
Similarly, there are many ways to *add batch normalization*,. You can add BN, to each convolution layer, or you can add some without adding some. I tried some ways, and the results were all good. Finally, **I chose to add BN to each convolution layer. This is also my final choice of the model**. Of course, the selection of full connection parameters and other operations also can be done, but after training observation found that the current model has a good effect, the subsequent attempt is no longer need, you can try it yourself.
"""
model = Sequential()
model.add(Conv2D(32, kernel_size=5, padding="same", activation='relu', kernel_initializer=glorot_uniform(seed=0), input_shape=(28,28,1)))
model.add(BatchNormalization(axis=3))
model.add(Conv2D(32, kernel_size=5, padding="same", activation='relu', kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization(axis=3))
model.add(Conv2D(32, kernel_size=5, padding="same", activation='relu', kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization(axis=3))
model.add(Conv2D(32, kernel_size=5, padding="same", activation='relu', kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization(axis=3))
model.add(MaxPooling2D(pool_size=3, padding="same"))
model.add(Conv2D(32, kernel_size=5, padding="same", activation='relu', kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization(axis=3))
model.add(MaxPooling2D(pool_size=3, padding="same"))
    
model.add(Conv2D(64, kernel_size=5, padding="same", activation='relu', kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization(axis=3))
model.add(Conv2D(64, kernel_size=5, padding="same", activation='relu', kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization(axis=3))
model.add(Conv2D(64, kernel_size=5, padding="same", activation='relu', kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization(axis=3))
model.add(Conv2D(64, kernel_size=5, padding="same", activation='relu', kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization(axis=3))
model.add(MaxPooling2D(pool_size=3, padding="same"))
model.add(Conv2D(64, kernel_size=5, padding="same", activation='relu', kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization(axis=3))
model.add(MaxPooling2D(pool_size=3, padding="same"))
    
model.add(Dropout(0.4))
    
model.add(Flatten())
model.add(Dense(256, activation="relu", kernel_initializer=glorot_uniform(seed=0)))
    
model.add(Dropout(0.5))
    
model.add(Dense(10, activation='softmax', kernel_initializer=glorot_uniform(seed=0)))
model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
print("---------------------------------------------------------------------------------------------")
print("加BN：")
start = time.time()
model.fit_generator(datagen.flow(train_x, train_y, batch_size=64), validation_data=(val_x, val_y), epochs=1, steps_per_epoch= len(train_x) // 64)
end = time.time()
print(str(end - start) + "秒")
print("---------------------------------------------------------------------------------------------")
"""
**Model training**
"""
"""
*1. Final model*
"""
model = Sequential()
model.add(Conv2D(32, kernel_size=5, padding="same", activation='relu', kernel_initializer=glorot_uniform(seed=0), input_shape=(28,28,1)))
model.add(BatchNormalization(axis=3))
model.add(Conv2D(32, kernel_size=5, padding="same", activation='relu', kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization(axis=3))
model.add(Conv2D(32, kernel_size=5, padding="same", activation='relu', kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization(axis=3))
model.add(Conv2D(32, kernel_size=5, padding="same", activation='relu', kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization(axis=3))
model.add(MaxPooling2D(pool_size=3, padding="same"))
model.add(Conv2D(32, kernel_size=5, padding="same", activation='relu', kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization(axis=3))
model.add(MaxPooling2D(pool_size=3, padding="same"))
    
model.add(Conv2D(64, kernel_size=5, padding="same", activation='relu', kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization(axis=3))
model.add(Conv2D(64, kernel_size=5, padding="same", activation='relu', kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization(axis=3))
model.add(Conv2D(64, kernel_size=5, padding="same", activation='relu', kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization(axis=3))
model.add(Conv2D(64, kernel_size=5, padding="same", activation='relu', kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization(axis=3))
model.add(MaxPooling2D(pool_size=3, padding="same"))
model.add(Conv2D(64, kernel_size=5, padding="same", activation='relu', kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization(axis=3))
model.add(MaxPooling2D(pool_size=3, padding="same"))
    
model.add(Dropout(0.4))
    
model.add(Flatten())
model.add(Dense(256, activation="relu", kernel_initializer=glorot_uniform(seed=0)))
    
model.add(Dropout(0.5))
    
model.add(Dense(10, activation='softmax', kernel_initializer=glorot_uniform(seed=0)))
"""
*2. Model compilation*
"""
model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
"""
*3. train*
"""
print("---------------------------------------------------------------------------------------------")
start = time.time()
model.fit_generator(datagen.flow(train_x, train_y, batch_size=64), validation_data=(val_x, val_y), epochs=1, steps_per_epoch= len(train_x) // 64)
end = time.time()
print(str(end - start) + "秒")
print("---------------------------------------------------------------------------------------------")
"""
*4. predict and submit*
"""
predict = np.argmax(model.predict(test_data), axis=1)
 
sample_submission["Label"] = predict
sample_submission.to_csv("1.csv", index=False)
"""
**Usually after 80 epochs of training, there will be some models that can make the score reach 0.996 +. These models can help us raise the score to 0.997+. I have obtained some excellent models. You can save the weights of these models.**
"""
"""
*5. Save model weights and load model weights*
"""
model.save_weights("my_model_weights_614_1.h5")
 
model.load_weights("my_model_weights_614_1.h5")
"""
**The model combination, using the simple voting method, using the model prediction of three scores of 0.996 +, basically can get the score of 0.997 +. Now take a model combination with a final score of 0.99728 as an example.**
"""
"""
Weights file can be downloaded from [here](https://pan.baidu.com/s/1T31ToVXSC7KWsSdcBjYkgQ%C2%A0#list/path=%2F), extraction code: **ltu5**
"""
"""
*load weight and Prediction*
"""
model.load_weights("my_model_weights_600_2.h5")
predict_600_2 = np.argmax(model.predict(test_data), axis=1)
 
model.load_weights("my_model_weights_628_2.h5")
predict_628_2 = np.argmax(model.predict(test_data), axis=1)
 
model.load_weights("my_model_weights_642_2.h5")
predict_642_2 = np.argmax(model.predict(test_data), axis=1)
"""
*Voting function*, did not find the kind of direct vote on the forecast, I do not know if there is, I wrote one of my own.
"""
def combine_model(predict1, predict2, predict3):
    not_equal_index = np.unique(np.hstack((np.where(predict1 != predict2)[0], np.where(predict1 != predict3)[0], np.where(predict2 != predict3)[0])))
    predict = np.copy(predict1)
    for i in not_equal_index:
        if (predict2[i] == predict3[i]) and (predict2[i] != predict1[i]):
            predict[i] = predict2[i]
    return predict
"""
*Combine predict*
"""
sample_submission["Label"] = combine_model(predict_642_2, predict_600_2, predict_628_2)
sample_submission.to_csv("1.csv", index=False)
"""
**In this way, I can get a score of 0.99728, which should be top10%. Of course, by combining different models, I may be able to achieve a higher score. I have a maximum score of 0.99785 through a variety of combinations. Those who are interested can continue to explore higher scores.**
"""