diff --git a/onnx/backend/test/runner/__init__.py b/onnx/backend/test/runner/__init__.py
index <HASH>..<HASH> 100644
--- a/onnx/backend/test/runner/__init__.py
+++ b/onnx/backend/test/runner/__init__.py
@@ -8,6 +8,7 @@ import functools
 import glob
 import os
 import re
+import shutil
 import tarfile
 import tempfile
 import unittest
@@ -151,7 +152,16 @@ class Runner(object):
         models_dir = os.getenv('ONNX_MODELS',
                                os.path.join(onnx_home, 'models'))
         model_dir = os.path.join(models_dir, model_test.model_name)
-        if not os.path.exists(model_dir):
+        if not os.path.exists(os.path.join(model_dir, 'model.onnx')):
+            if os.path.exists(model_dir):
+                bi = 0
+                while True:
+                    dest = '{}.old.{}'.format(model_dir, bi)
+                    if os.path.exists(dest):
+                        bi += 1
+                        continue
+                    shutil.move(model_dir, dest)
+                    break
             os.makedirs(model_dir)
             url = 'https://s3.amazonaws.com/download.onnx/models/{}.tar.gz'.format(
                 model_test.model_name)
Change the cached model checking logic (#<I>)

* Change the cached model checking logic



* Move the old folder, instead of deleting