<p dir="auto">Seems to be an issue serializing a model returned from<br>
tensorflow-gpu==1.11.0<br>
keras==2.2.4</p>
<p dir="auto">from keras.utils.training_utils import multi_gpu_model<br>
..<br>
..</p>
<p dir="auto">gpu_model = multi_gpu_model(model,gpus=2)<br>
gpu_model.compile(loss="binary_crossentropy", optimizer=opt,metrics=["accuracy"])<br>
..<br>
..<br>
fit<br>
..</p>
<p dir="auto">print("[INFO] serializing model...")<br>
gpu_model.save(config.MODEL_PATH, overwrite=True)</p>
<p dir="auto">[INFO] serializing model...<br>
Traceback (most recent call last):<br>
File "train_alexnet.py", line 70, in <br>
gpu_model.save(config.MODEL_PATH, overwrite=True)<br>
File "C:\Users\ianfe\Envs\dl4cv\lib\site-packages\keras\engine\network.py", line 1090, in save<br>
save_model(self, filepath, overwrite, include_optimizer)<br>
File "C:\Users\ianfe\Envs\dl4cv\lib\site-packages\keras\engine\saving.py", line 382, in save_model<br>
_serialize_model(model, f, include_optimizer)<br>
File "C:\Users\ianfe\Envs\dl4cv\lib\site-packages\keras\engine\saving.py", line 83, in _serialize_model<br>
model_config['config'] = model.get_config()<br>
File "C:\Users\ianfe\Envs\dl4cv\lib\site-packages\keras\engine\network.py", line 931, in get_config<br>
return copy.deepcopy(config)<br>
File "C:\Users\ianfe\Envs\dl4cv\lib\copy.py", line 150, in deepcopy<br>
y = copier(x, memo)<br>
File "C:\Users\ianfe\Envs\dl4cv\lib\copy.py", line 240, in _deepcopy_dict<br>
y[deepcopy(key, memo)] = deepcopy(value, memo)<br>
File "C:\Users\ianfe\Envs\dl4cv\lib\copy.py", line 150, in deepcopy<br>
y = copier(x, memo)<br>
File "C:\Users\ianfe\Envs\dl4cv\lib\copy.py", line 215, in _deepcopy_list<br>
append(deepcopy(a, memo))<br>
File "C:\Users\ianfe\Envs\dl4cv\lib\copy.py", line 150, in deepcopy<br>
y = copier(x, memo)<br>
File "C:\Users\ianfe\Envs\dl4cv\lib\copy.py", line 240, in _deepcopy_dict<br>
y[deepcopy(key, memo)] = deepcopy(value, memo)<br>
File "C:\Users\ianfe\Envs\dl4cv\lib\copy.py", line 150, in deepcopy<br>
y = copier(x, memo)<br>
File "C:\Users\ianfe\Envs\dl4cv\lib\copy.py", line 240, in _deepcopy_dict<br>
y[deepcopy(key, memo)] = deepcopy(value, memo)<br>
File "C:\Users\ianfe\Envs\dl4cv\lib\copy.py", line 150, in deepcopy<br>
y = copier(x, memo)<br>
File "C:\Users\ianfe\Envs\dl4cv\lib\copy.py", line 220, in _deepcopy_tuple<br>
y = [deepcopy(a, memo) for a in x]<br>
File "C:\Users\ianfe\Envs\dl4cv\lib\copy.py", line 220, in <br>
y = [deepcopy(a, memo) for a in x]<br>
File "C:\Users\ianfe\Envs\dl4cv\lib\copy.py", line 150, in deepcopy<br>
y = copier(x, memo)<br>
File "C:\Users\ianfe\Envs\dl4cv\lib\copy.py", line 220, in _deepcopy_tuple<br>
y = [deepcopy(a, memo) for a in x]<br>
File "C:\Users\ianfe\Envs\dl4cv\lib\copy.py", line 220, in <br>
y = [deepcopy(a, memo) for a in x]<br>
File "C:\Users\ianfe\Envs\dl4cv\lib\copy.py", line 169, in deepcopy<br>
rv = reductor(4)<br>
TypeError: can't pickle module objects</p>
<p dir="auto">Hey guys,</p>
<p dir="auto">the new multi_gpu feature seems to have a bug. If you want to save the model you get an error like the one below. To reproduce just run the test multi_gpu_test_simple_model() with parallel_model.save("logs/model.h5") at the end.</p>
<div class="highlight highlight-source-python notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="def multi_gpu_test_simple_model():
    print('####### test simple model')
    num_samples = 1000
    input_dim = 10
    output_dim = 1
    hidden_dim = 10
    gpus = 4
    epochs = 2
    model = keras.models.Sequential()
    model.add(keras.layers.Dense(hidden_dim,
                                 input_shape=(input_dim,)))
    model.add(keras.layers.Dense(output_dim))

    x = np.random.random((num_samples, input_dim))
    y = np.random.random((num_samples, output_dim))
    parallel_model = multi_gpu_model(model, gpus=gpus)

    parallel_model.compile(loss='mse', optimizer='rmsprop')
    parallel_model.fit(x, y, epochs=epochs)

    parallel_model.save(&quot;logs/model.h5&quot;)


multi_gpu_test_simple_model()"><pre class="notranslate"><span class="pl-k">def</span> <span class="pl-en">multi_gpu_test_simple_model</span>():
    <span class="pl-en">print</span>(<span class="pl-s">'####### test simple model'</span>)
    <span class="pl-s1">num_samples</span> <span class="pl-c1">=</span> <span class="pl-c1">1000</span>
    <span class="pl-s1">input_dim</span> <span class="pl-c1">=</span> <span class="pl-c1">10</span>
    <span class="pl-s1">output_dim</span> <span class="pl-c1">=</span> <span class="pl-c1">1</span>
    <span class="pl-s1">hidden_dim</span> <span class="pl-c1">=</span> <span class="pl-c1">10</span>
    <span class="pl-s1">gpus</span> <span class="pl-c1">=</span> <span class="pl-c1">4</span>
    <span class="pl-s1">epochs</span> <span class="pl-c1">=</span> <span class="pl-c1">2</span>
    <span class="pl-s1">model</span> <span class="pl-c1">=</span> <span class="pl-s1">keras</span>.<span class="pl-s1">models</span>.<span class="pl-v">Sequential</span>()
    <span class="pl-s1">model</span>.<span class="pl-en">add</span>(<span class="pl-s1">keras</span>.<span class="pl-s1">layers</span>.<span class="pl-v">Dense</span>(<span class="pl-s1">hidden_dim</span>,
                                 <span class="pl-s1">input_shape</span><span class="pl-c1">=</span>(<span class="pl-s1">input_dim</span>,)))
    <span class="pl-s1">model</span>.<span class="pl-en">add</span>(<span class="pl-s1">keras</span>.<span class="pl-s1">layers</span>.<span class="pl-v">Dense</span>(<span class="pl-s1">output_dim</span>))

    <span class="pl-s1">x</span> <span class="pl-c1">=</span> <span class="pl-s1">np</span>.<span class="pl-s1">random</span>.<span class="pl-en">random</span>((<span class="pl-s1">num_samples</span>, <span class="pl-s1">input_dim</span>))
    <span class="pl-s1">y</span> <span class="pl-c1">=</span> <span class="pl-s1">np</span>.<span class="pl-s1">random</span>.<span class="pl-en">random</span>((<span class="pl-s1">num_samples</span>, <span class="pl-s1">output_dim</span>))
    <span class="pl-s1">parallel_model</span> <span class="pl-c1">=</span> <span class="pl-en">multi_gpu_model</span>(<span class="pl-s1">model</span>, <span class="pl-s1">gpus</span><span class="pl-c1">=</span><span class="pl-s1">gpus</span>)

    <span class="pl-s1">parallel_model</span>.<span class="pl-en">compile</span>(<span class="pl-s1">loss</span><span class="pl-c1">=</span><span class="pl-s">'mse'</span>, <span class="pl-s1">optimizer</span><span class="pl-c1">=</span><span class="pl-s">'rmsprop'</span>)
    <span class="pl-s1">parallel_model</span>.<span class="pl-en">fit</span>(<span class="pl-s1">x</span>, <span class="pl-s1">y</span>, <span class="pl-s1">epochs</span><span class="pl-c1">=</span><span class="pl-s1">epochs</span>)

    <span class="pl-s1">parallel_model</span>.<span class="pl-en">save</span>(<span class="pl-s">"logs/model.h5"</span>)


<span class="pl-en">multi_gpu_test_simple_model</span>()</pre></div>
<blockquote>
<p dir="auto">1000/1000 [==============================] - ETA: 0s - loss: 0.4537<br>
Epoch 2/2<br>
1000/1000 [==============================] - ETA: 0s - loss: 0.2939<br>
Traceback (most recent call last):<br>
File "steps_kt/test.py", line 43, in <br>
multi_gpu_test_simple_model()<br>
File "steps_kt/test.py", line 40, in multi_gpu_test_simple_model<br>
parallel_model.save("logs/model.h5")<br>
File "/home/y0052080/pyenv/lib/python3.6/site-packages/Keras-2.0.8-py3.6.egg/keras/engine/topology.py", line 2555, in save<br>
File "/home/y0052080/pyenv/lib/python3.6/site-packages/Keras-2.0.8-py3.6.egg/keras/models.py", line 107, in save_model<br>
File "/home/y0052080/pyenv/lib/python3.6/site-packages/Keras-2.0.8-py3.6.egg/keras/engine/topology.py", line 2396, in get_config<br>
File "/cluster/tools/python3/lib/python3.6/copy.py", line 150, in deepcopy<br>
y = copier(x, memo)<br>
File "/cluster/tools/python3/lib/python3.6/copy.py", line 240, in _deepcopy_dict<br>
y[deepcopy(key, memo)] = deepcopy(value, memo)<br>
File "/cluster/tools/python3/lib/python3.6/copy.py", line 150, in deepcopy<br>
y = copier(x, memo)<br>
File "/cluster/tools/python3/lib/python3.6/copy.py", line 215, in _deepcopy_list<br>
append(deepcopy(a, memo))<br>
File "/cluster/tools/python3/lib/python3.6/copy.py", line 150, in deepcopy<br>
y = copier(x, memo)<br>
File "/cluster/tools/python3/lib/python3.6/copy.py", line 240, in _deepcopy_dict<br>
y[deepcopy(key, memo)] = deepcopy(value, memo)<br>
File "/cluster/tools/python3/lib/python3.6/copy.py", line 150, in deepcopy<br>
y = copier(x, memo)<br>
File "/cluster/tools/python3/lib/python3.6/copy.py", line 240, in _deepcopy_dict<br>
y[deepcopy(key, memo)] = deepcopy(value, memo)<br>
File "/cluster/tools/python3/lib/python3.6/copy.py", line 150, in deepcopy<br>
y = copier(x, memo)<br>
File "/cluster/tools/python3/lib/python3.6/copy.py", line 220, in _deepcopy_tuple<br>
y = [deepcopy(a, memo) for a in x]<br>
File "/cluster/tools/python3/lib/python3.6/copy.py", line 220, in <br>
y = [deepcopy(a, memo) for a in x]<br>
File "/cluster/tools/python3/lib/python3.6/copy.py", line 150, in deepcopy<br>
y = copier(x, memo)<br>
File "/cluster/tools/python3/lib/python3.6/copy.py", line 220, in _deepcopy_tuple<br>
y = [deepcopy(a, memo) for a in x]<br>
File "/cluster/tools/python3/lib/python3.6/copy.py", line 220, in <br>
y = [deepcopy(a, memo) for a in x]<br>
File "/cluster/tools/python3/lib/python3.6/copy.py", line 169, in deepcopy<br>
rv = reductor(4)<br>
TypeError: can't pickle module objects</p>
</blockquote>
<p dir="auto">Please make sure that the boxes below are checked before you submit your issue. If your issue is an implementation question, please ask your question on <a href="http://stackoverflow.com/questions/tagged/keras" rel="nofollow">StackOverflow</a> or <a href="https://keras-slack-autojoin.herokuapp.com/" rel="nofollow">join the Keras Slack channel</a> and ask there instead of filing a GitHub issue.</p>
<p dir="auto">Thank you!</p>
<ul dir="auto">
<li>
<p dir="auto">[ x] Check that you are up-to-date with the master branch of Keras. You can update with:<br>
pip install git+git://github.com/fchollet/keras.git --upgrade --no-deps</p>
</li>
<li>
<p dir="auto">[ x] If running on TensorFlow, check that you are up-to-date with the latest version. The installation instructions can be found <a href="https://www.tensorflow.org/get_started/os_setup" rel="nofollow">here</a>.</p>
</li>
</ul>