prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  SDK can't create and watch inferenceservice at the same time
  /kind bug
  
  **What steps did you take and what happened:**
  Running the below python SDK code to create and watch the inferenceservice occasionally have race condition on getting the status.
  ```python
  KFServing.create(isvc, watch=True, timeout_seconds=120)
  ```
  
  ```python
  ---------------------------------------------------------------------------
  KeyError                                  Traceback (most recent call last)
  <ipython-input-4-06b315a74aac> in <module>
        1 KFServing = KFServingClient()
  ----> 2 KFServing.create(isvc, watch=True, timeout_seconds=120)
  
  ~/.local/lib/python3.6/site-packages/kfserving/api/kf_serving_client.py in create(self, inferenceservice, namespace, watch, timeout_seconds)
      114                 name=outputs['metadata']['name'],
      115                 namespace=namespace,
  --> 116                 timeout_seconds=timeout_seconds)
      117         else:
      118             return outputs
  
  ~/.local/lib/python3.6/site-packages/kfserving/api/kf_serving_watch.py in watch(name, namespace, timeout_seconds)
       46             continue
       47         else:
  ---> 48             url = isvc['status'].get('url', '')
       49             default_traffic = isvc['status'].get('traffic', '')
       50             canary_traffic = isvc['status'].get('canaryTraffic', '')
  
  KeyError: 'status'
  ```
  **What did you expect to happen:**
  it should watch the inferenceservice without error out on race condition.
  
  
  **Anything else you would like to add:**
  This error happens way more often on kfserving SDK 0.3.0+
  
  
  **Environment:**
  
  - Istio Version: 1.1.6
  - Knative Version: 1.11.2
  - KFServing Version: 0.3.0
  - Kubeflow version: 1.0.2
  - Kfdef:[k8s_istio/istio_dex/gcp_basic_auth/gcp_iap/aws/aws_cognito/ibm]
  - Minikube version:
  - Kubernetes version: (use `kubectl version`): 1.15
  - OS (e.g. from `/etc/os-release`):
  
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `python/kfserving/kfserving/api/kf_serving_watch.py`
  Content:
  ```
  1 # Copyright 2019 The Kubeflow Authors.
  2 #
  3 # Licensed under the Apache License, Version 2.0 (the "License");
  4 # you may not use this file except in compliance with the License.
  5 # You may obtain a copy of the License at
  6 #
  7 #     http://www.apache.org/licenses/LICENSE-2.0
  8 #
  9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 
  15 from kubernetes import client
  16 from kubernetes import watch as k8s_watch
  17 from table_logger import TableLogger
  18 
  19 from ..constants import constants
  20 from ..utils import utils
  21 
  22 
  23 def watch(name=None, namespace=None, timeout_seconds=600):
  24     """Watch the created or patched InferenceService in the specified namespace"""
  25 
  26     if namespace is None:
  27         namespace = utils.get_default_target_namespace()
  28 
  29     tbl = TableLogger(
  30         columns='NAME,READY,DEFAULT_TRAFFIC,CANARY_TRAFFIC,URL',
  31         colwidth={'NAME': 20, 'READY':10, 'DEFAULT_TRAFFIC':15, 'CANARY_TRAFFIC':15, 'URL': 50},
  32         border=False)
  33 
  34     stream = k8s_watch.Watch().stream(
  35         client.CustomObjectsApi().list_namespaced_custom_object,
  36         constants.KFSERVING_GROUP,
  37         constants.KFSERVING_VERSION,
  38         namespace,
  39         constants.KFSERVING_PLURAL,
  40         timeout_seconds=timeout_seconds)
  41 
  42     for event in stream:
  43         isvc = event['object']
  44         isvc_name = isvc['metadata']['name']
  45         if name and name != isvc_name:
  46             continue
  47         else:
  48             url = isvc['status'].get('url', '')
  49             default_traffic = isvc['status'].get('traffic', '')
  50             canary_traffic = isvc['status'].get('canaryTraffic', '')
  51             status = 'Unknown'
  52             for condition in isvc['status'].get('conditions', {}):
  53                 if condition.get('type', '') == 'Ready':
  54                     status = condition.get('status', 'Unknown')
  55             tbl(isvc_name, status, default_traffic, canary_traffic, url)
  56 
  57             if name == isvc_name and status == 'True':
  58                 break
  59
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/python/kfserving/kfserving/api/kf_serving_watch.py b/python/kfserving/kfserving/api/kf_serving_watch.py
  --- a/python/kfserving/kfserving/api/kf_serving_watch.py
  +++ b/python/kfserving/kfserving/api/kf_serving_watch.py
  @@ -12,6 +12,7 @@
   # See the License for the specific language governing permissions and
   # limitations under the License.
   
  +import time
   from kubernetes import client
   from kubernetes import watch as k8s_watch
   from table_logger import TableLogger
  @@ -45,14 +46,20 @@
           if name and name != isvc_name:
               continue
           else:
  -            url = isvc['status'].get('url', '')
  -            default_traffic = isvc['status'].get('traffic', '')
  -            canary_traffic = isvc['status'].get('canaryTraffic', '')
  -            status = 'Unknown'
  -            for condition in isvc['status'].get('conditions', {}):
  -                if condition.get('type', '') == 'Ready':
  -                    status = condition.get('status', 'Unknown')
  -            tbl(isvc_name, status, default_traffic, canary_traffic, url)
  +            if isvc.get('status', ''):
  +                url = isvc['status'].get('url', '')
  +                default_traffic = isvc['status'].get('traffic', '')
  +                canary_traffic = isvc['status'].get('canaryTraffic', '')
  +                status = 'Unknown'
  +                for condition in isvc['status'].get('conditions', {}):
  +                    if condition.get('type', '') == 'Ready':
  +                        status = condition.get('status', 'Unknown')
  +                tbl(isvc_name, status, default_traffic, canary_traffic, url)
  +            else:
  +                tbl(isvc_name, 'Unknown', '', '', '')
  +                # Sleep 2 to avoid status section is not generated within a very short time.
  +                time.sleep(2)
  +                continue
   
               if name == isvc_name and status == 'True':
                   break
  
verification_info: {"golden_diff": "diff --git a/python/kfserving/kfserving/api/kf_serving_watch.py b/python/kfserving/kfserving/api/kf_serving_watch.py\n--- a/python/kfserving/kfserving/api/kf_serving_watch.py\n+++ b/python/kfserving/kfserving/api/kf_serving_watch.py\n@@ -12,6 +12,7 @@\n # See the License for the specific language governing permissions and\n # limitations under the License.\n \n+import time\n from kubernetes import client\n from kubernetes import watch as k8s_watch\n from table_logger import TableLogger\n@@ -45,14 +46,20 @@\n         if name and name != isvc_name:\n             continue\n         else:\n-            url = isvc['status'].get('url', '')\n-            default_traffic = isvc['status'].get('traffic', '')\n-            canary_traffic = isvc['status'].get('canaryTraffic', '')\n-            status = 'Unknown'\n-            for condition in isvc['status'].get('conditions', {}):\n-                if condition.get('type', '') == 'Ready':\n-                    status = condition.get('status', 'Unknown')\n-            tbl(isvc_name, status, default_traffic, canary_traffic, url)\n+            if isvc.get('status', ''):\n+                url = isvc['status'].get('url', '')\n+                default_traffic = isvc['status'].get('traffic', '')\n+                canary_traffic = isvc['status'].get('canaryTraffic', '')\n+                status = 'Unknown'\n+                for condition in isvc['status'].get('conditions', {}):\n+                    if condition.get('type', '') == 'Ready':\n+                        status = condition.get('status', 'Unknown')\n+                tbl(isvc_name, status, default_traffic, canary_traffic, url)\n+            else:\n+                tbl(isvc_name, 'Unknown', '', '', '')\n+                # Sleep 2 to avoid status section is not generated within a very short time.\n+                time.sleep(2)\n+                continue\n \n             if name == isvc_name and status == 'True':\n                 break\n", "issue": "SDK can't create and watch inferenceservice at the same time\n/kind bug\r\n\r\n**What steps did you take and what happened:**\r\nRunning the below python SDK code to create and watch the inferenceservice occasionally have race condition on getting the status.\r\n```python\r\nKFServing.create(isvc, watch=True, timeout_seconds=120)\r\n```\r\n\r\n```python\r\n---------------------------------------------------------------------------\r\nKeyError                                  Traceback (most recent call last)\r\n<ipython-input-4-06b315a74aac> in <module>\r\n      1 KFServing = KFServingClient()\r\n----> 2 KFServing.create(isvc, watch=True, timeout_seconds=120)\r\n\r\n~/.local/lib/python3.6/site-packages/kfserving/api/kf_serving_client.py in create(self, inferenceservice, namespace, watch, timeout_seconds)\r\n    114                 name=outputs['metadata']['name'],\r\n    115                 namespace=namespace,\r\n--> 116                 timeout_seconds=timeout_seconds)\r\n    117         else:\r\n    118             return outputs\r\n\r\n~/.local/lib/python3.6/site-packages/kfserving/api/kf_serving_watch.py in watch(name, namespace, timeout_seconds)\r\n     46             continue\r\n     47         else:\r\n---> 48             url = isvc['status'].get('url', '')\r\n     49             default_traffic = isvc['status'].get('traffic', '')\r\n     50             canary_traffic = isvc['status'].get('canaryTraffic', '')\r\n\r\nKeyError: 'status'\r\n```\r\n**What did you expect to happen:**\r\nit should watch the inferenceservice without error out on race condition.\r\n\r\n\r\n**Anything else you would like to add:**\r\nThis error happens way more often on kfserving SDK 0.3.0+\r\n\r\n\r\n**Environment:**\r\n\r\n- Istio Version: 1.1.6\r\n- Knative Version: 1.11.2\r\n- KFServing Version: 0.3.0\r\n- Kubeflow version: 1.0.2\r\n- Kfdef:[k8s_istio/istio_dex/gcp_basic_auth/gcp_iap/aws/aws_cognito/ibm]\r\n- Minikube version:\r\n- Kubernetes version: (use `kubectl version`): 1.15\r\n- OS (e.g. from `/etc/os-release`):\r\n\n", "before_files": [{"content": "# Copyright 2019 The Kubeflow Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom kubernetes import client\nfrom kubernetes import watch as k8s_watch\nfrom table_logger import TableLogger\n\nfrom ..constants import constants\nfrom ..utils import utils\n\n\ndef watch(name=None, namespace=None, timeout_seconds=600):\n    \"\"\"Watch the created or patched InferenceService in the specified namespace\"\"\"\n\n    if namespace is None:\n        namespace = utils.get_default_target_namespace()\n\n    tbl = TableLogger(\n        columns='NAME,READY,DEFAULT_TRAFFIC,CANARY_TRAFFIC,URL',\n        colwidth={'NAME': 20, 'READY':10, 'DEFAULT_TRAFFIC':15, 'CANARY_TRAFFIC':15, 'URL': 50},\n        border=False)\n\n    stream = k8s_watch.Watch().stream(\n        client.CustomObjectsApi().list_namespaced_custom_object,\n        constants.KFSERVING_GROUP,\n        constants.KFSERVING_VERSION,\n        namespace,\n        constants.KFSERVING_PLURAL,\n        timeout_seconds=timeout_seconds)\n\n    for event in stream:\n        isvc = event['object']\n        isvc_name = isvc['metadata']['name']\n        if name and name != isvc_name:\n            continue\n        else:\n            url = isvc['status'].get('url', '')\n            default_traffic = isvc['status'].get('traffic', '')\n            canary_traffic = isvc['status'].get('canaryTraffic', '')\n            status = 'Unknown'\n            for condition in isvc['status'].get('conditions', {}):\n                if condition.get('type', '') == 'Ready':\n                    status = condition.get('status', 'Unknown')\n            tbl(isvc_name, status, default_traffic, canary_traffic, url)\n\n            if name == isvc_name and status == 'True':\n                break\n", "path": "python/kfserving/kfserving/api/kf_serving_watch.py"}], "after_files": [{"content": "# Copyright 2019 The Kubeflow Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport time\nfrom kubernetes import client\nfrom kubernetes import watch as k8s_watch\nfrom table_logger import TableLogger\n\nfrom ..constants import constants\nfrom ..utils import utils\n\n\ndef watch(name=None, namespace=None, timeout_seconds=600):\n    \"\"\"Watch the created or patched InferenceService in the specified namespace\"\"\"\n\n    if namespace is None:\n        namespace = utils.get_default_target_namespace()\n\n    tbl = TableLogger(\n        columns='NAME,READY,DEFAULT_TRAFFIC,CANARY_TRAFFIC,URL',\n        colwidth={'NAME': 20, 'READY':10, 'DEFAULT_TRAFFIC':15, 'CANARY_TRAFFIC':15, 'URL': 50},\n        border=False)\n\n    stream = k8s_watch.Watch().stream(\n        client.CustomObjectsApi().list_namespaced_custom_object,\n        constants.KFSERVING_GROUP,\n        constants.KFSERVING_VERSION,\n        namespace,\n        constants.KFSERVING_PLURAL,\n        timeout_seconds=timeout_seconds)\n\n    for event in stream:\n        isvc = event['object']\n        isvc_name = isvc['metadata']['name']\n        if name and name != isvc_name:\n            continue\n        else:\n            if isvc.get('status', ''):\n                url = isvc['status'].get('url', '')\n                default_traffic = isvc['status'].get('traffic', '')\n                canary_traffic = isvc['status'].get('canaryTraffic', '')\n                status = 'Unknown'\n                for condition in isvc['status'].get('conditions', {}):\n                    if condition.get('type', '') == 'Ready':\n                        status = condition.get('status', 'Unknown')\n                tbl(isvc_name, status, default_traffic, canary_traffic, url)\n            else:\n                tbl(isvc_name, 'Unknown', '', '', '')\n                # Sleep 2 to avoid status section is not generated within a very short time.\n                time.sleep(2)\n                continue\n\n            if name == isvc_name and status == 'True':\n                break\n", "path": "python/kfserving/kfserving/api/kf_serving_watch.py"}]}