We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
KServe lgbserver runtime error input for v1 endpoint
/kind bug

**What steps did you take and what happened:**
lgbserver used to support both of following inputs formats prior to 0.11 release.

```python
request = {'inputs' : [{'sepal_width_(cm)': {0: 3.5}, 
'petal_length_(cm)': {0: 1.4}, 
'petal_width_(cm)': {0: 0.2},
'sepal_length_(cm)': {0: 5.1} }]}
```

```python
request2 = {'inputs': [
[{'sepal_width_(cm)': 3.5}, 
 {'petal_length_(cm)': 1.4}, 
 {'petal_width_(cm)': 0.2}, 
 {'sepal_length_(cm)': 5.1}]
 ] }
```

KServe only documented the first input format https://kserve.github.io/website/0.11/modelserving/v1beta1/lightgbm

and in 0.11 the second input format stop working with following error

```bash
2023-11-03 09:06:02.099 32367 kserve ERROR [inference_error_handler():89] Exception:
Traceback (most recent call last):
  File "/Users/dsun20/kserve/python/lgbserver/lgbserver/model.py", line 62, in predict
    result = self._booster.predict(instances)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/homebrew/lib/python3.11/site-packages/lightgbm/basic.py", line 4220, in predict
    return predictor.predict(
           ^^^^^^^^^^^^^^^^^^
  File "/opt/homebrew/lib/python3.11/site-packages/lightgbm/basic.py", line 1047, in predict
    preds, nrow = self.__pred_for_np2d(
                  ^^^^^^^^^^^^^^^^^^^^^
  File "/opt/homebrew/lib/python3.11/site-packages/lightgbm/basic.py", line 1187, in __pred_for_np2d
    return self.__inner_predict_np2d(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/homebrew/lib/python3.11/site-packages/lightgbm/basic.py", line 1127, in __inner_predict_np2d
    data = np.array(mat.reshape(mat.size), dtype=np.float32)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
TypeError: float() argument must be a string or a real number, not 'dict'
```


**What did you expect to happen:**


**What's the InferenceService yaml:**
[To help us debug please run `kubectl get isvc $name -n $namespace -oyaml` and paste the output]

**Anything else you would like to add:**
[Miscellaneous information that will assist in solving the issue.]


**Environment:**

- Istio Version:
- Knative Version:
- KServe Version:
- Kubeflow version:
- Cloud Environment:[k8s_istio/istio_dex/gcp_basic_auth/gcp_iap/aws/aws_cognito/ibm]
- Minikube/Kind version:
- Kubernetes version: (use `kubectl version`):
- OS (e.g. from `/etc/os-release`):


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `python/kserve/kserve/utils/utils.py`
Content:
```
1 # Copyright 2021 The KServe Authors.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #    http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 
15 import os
16 import sys
17 import uuid
18 from kserve.protocol.grpc.grpc_predict_v2_pb2 import InferParameter
19 from typing import Dict, Union
20 
21 from kserve.utils.numpy_codec import from_np_dtype
22 import pandas as pd
23 import numpy as np
24 import psutil
25 from cloudevents.conversion import to_binary, to_structured
26 from cloudevents.http import CloudEvent
27 from grpc import ServicerContext
28 from kserve.protocol.infer_type import InferOutput, InferRequest, InferResponse
29 
30 
31 def is_running_in_k8s():
32     return os.path.isdir('/var/run/secrets/kubernetes.io/')
33 
34 
35 def get_current_k8s_namespace():
36     with open('/var/run/secrets/kubernetes.io/serviceaccount/namespace', 'r') as f:
37         return f.readline()
38 
39 
40 def get_default_target_namespace():
41     if not is_running_in_k8s():
42         return 'default'
43     return get_current_k8s_namespace()
44 
45 
46 def get_isvc_namespace(inferenceservice):
47     return inferenceservice.metadata.namespace or get_default_target_namespace()
48 
49 
50 def get_ig_namespace(inferencegraph):
51     return inferencegraph.metadata.namespace or get_default_target_namespace()
52 
53 
54 def cpu_count():
55     """Get the available CPU count for this system.
56     Takes the minimum value from the following locations:
57     - Total system cpus available on the host.
58     - CPU Affinity (if set)
59     - Cgroups limit (if set)
60     """
61     count = os.cpu_count()
62 
63     # Check CPU affinity if available
64     try:
65         affinity_count = len(psutil.Process().cpu_affinity())
66         if affinity_count > 0:
67             count = min(count, affinity_count)
68     except Exception:
69         pass
70 
71     # Check cgroups if available
72     if sys.platform == "linux":
73         try:
74             with open("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us") as f:
75                 quota = int(f.read())
76             with open("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_period_us") as f:
77                 period = int(f.read())
78             cgroups_count = int(quota / period)
79             if cgroups_count > 0:
80                 count = min(count, cgroups_count)
81         except Exception:
82             pass
83 
84     return count
85 
86 
87 def is_structured_cloudevent(body: Dict) -> bool:
88     """Returns True if the JSON request body resembles a structured CloudEvent"""
89     return "time" in body \
90            and "type" in body \
91            and "source" in body \
92            and "id" in body \
93            and "specversion" in body \
94            and "data" in body
95 
96 
97 def create_response_cloudevent(model_name: str, response: Dict, req_attributes: Dict,
98                                binary_event=False) -> tuple:
99     ce_attributes = {}
100 
101     if os.getenv("CE_MERGE", "false").lower() == "true":
102         if binary_event:
103             ce_attributes = req_attributes
104             if "datacontenttype" in ce_attributes:  # Optional field so must check
105                 del ce_attributes["datacontenttype"]
106         else:
107             ce_attributes = req_attributes
108 
109         # Remove these fields so we generate new ones
110         del ce_attributes["id"]
111         del ce_attributes["time"]
112 
113     ce_attributes["type"] = os.getenv("CE_TYPE", "io.kserve.inference.response")
114     ce_attributes["source"] = os.getenv("CE_SOURCE", f"io.kserve.inference.{model_name}")
115 
116     event = CloudEvent(ce_attributes, response)
117 
118     if binary_event:
119         event_headers, event_body = to_binary(event)
120     else:
121         event_headers, event_body = to_structured(event)
122 
123     return event_headers, event_body
124 
125 
126 def generate_uuid() -> str:
127     return str(uuid.uuid4())
128 
129 
130 def to_headers(context: ServicerContext) -> Dict[str, str]:
131     metadata = context.invocation_metadata()
132     if hasattr(context, "trailing_metadata"):
133         metadata += context.trailing_metadata()
134     headers = {}
135     for metadatum in metadata:
136         headers[metadatum.key] = metadatum.value
137 
138     return headers
139 
140 
141 def get_predict_input(payload: Union[Dict, InferRequest]) -> Union[np.ndarray, pd.DataFrame]:
142     if isinstance(payload, Dict):
143         instances = payload["inputs"] if "inputs" in payload else payload["instances"]
144         if len(instances) == 0:
145             return np.array(instances)
146         if isinstance(instances[0], Dict):
147             dfs = []
148             for input in instances:
149                 dfs.append(pd.DataFrame(input))
150             inputs = pd.concat(dfs, axis=0)
151             return inputs
152         else:
153             return np.array(instances)
154 
155     elif isinstance(payload, InferRequest):
156         content_type = ''
157         parameters = payload.parameters
158         if parameters:
159             if isinstance(parameters.get("content_type"), InferParameter):
160                 # for v2 grpc, we get InferParameter obj eg: {"content_type": string_param: "pd"}
161                 content_type = str(parameters.get("content_type").string_param)
162             else:
163                 # for v2 http, we get string eg: {"content_type": "pd"}
164                 content_type = parameters.get("content_type")
165 
166         if content_type == "pd":
167             return payload.as_dataframe()
168         else:
169             input = payload.inputs[0]
170             return input.as_numpy()
171 
172 
173 def get_predict_response(payload: Union[Dict, InferRequest], result: Union[np.ndarray, pd.DataFrame],
174                          model_name: str) -> Union[Dict, InferResponse]:
175     if isinstance(payload, Dict):
176         infer_outputs = result
177         if isinstance(result, pd.DataFrame):
178             infer_outputs = []
179             for label, row in result.iterrows():
180                 infer_outputs.append(row.to_dict())
181         elif isinstance(result, np.ndarray):
182             infer_outputs = result.tolist()
183         return {"predictions": infer_outputs}
184     elif isinstance(payload, InferRequest):
185         infer_outputs = []
186         if isinstance(result, pd.DataFrame):
187             for col in result.columns:
188                 infer_output = InferOutput(
189                     name=col,
190                     shape=list(result[col].shape),
191                     datatype=from_np_dtype(result[col].dtype),
192                     data=result[col].tolist()
193                 )
194                 infer_outputs.append(infer_output)
195         else:
196             infer_output = InferOutput(
197                 name="output-0",
198                 shape=list(result.shape),
199                 datatype=from_np_dtype(result.dtype),
200                 data=result.flatten().tolist()
201             )
202             infer_outputs.append(infer_output)
203         return InferResponse(
204             model_name=model_name,
205             infer_outputs=infer_outputs,
206             response_id=payload.id if payload.id else generate_uuid()
207         )
208 
209 
210 def strtobool(val: str) -> bool:
211     """Convert a string representation of truth to True or False.
212 
213     True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
214     are 'n', 'no', 'f', 'false', 'off', and '0'.  Raises ValueError if
215     'val' is anything else.
216 
217     Adapted from deprecated `distutils`
218     https://github.com/python/cpython/blob/3.11/Lib/distutils/util.py
219     """
220     val = val.lower()
221     if val in ('y', 'yes', 't', 'true', 'on', '1'):
222         return True
223     elif val in ('n', 'no', 'f', 'false', 'off', '0'):
224         return False
225     else:
226         raise ValueError("invalid truth value %r" % (val,))
227
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/python/kserve/kserve/utils/utils.py b/python/kserve/kserve/utils/utils.py
--- a/python/kserve/kserve/utils/utils.py
+++ b/python/kserve/kserve/utils/utils.py
@@ -16,7 +16,7 @@
 import sys
 import uuid
 from kserve.protocol.grpc.grpc_predict_v2_pb2 import InferParameter
-from typing import Dict, Union
+from typing import Dict, Union, List
 
 from kserve.utils.numpy_codec import from_np_dtype
 import pandas as pd
@@ -149,6 +149,24 @@
                 dfs.append(pd.DataFrame(input))
             inputs = pd.concat(dfs, axis=0)
             return inputs
+
+        # Handles the following input format
+        # {'inputs': [
+        #     [{'sepal_width_(cm)': 3.5},
+        #      {'petal_length_(cm)': 1.4},
+        #      {'petal_width_(cm)': 0.2},
+        #      {'sepal_length_(cm)': 5.1}]
+        # ]}
+        elif isinstance(instances[0], List) and len(instances[0]) != 0 and isinstance(instances[0][0], Dict):
+            data: Dict[str, List] = {}
+            for instance in instances:
+                for item in instance:
+                    for key, val in item.items():
+                        if key in data:
+                            data[key].append(val)
+                        else:
+                            data[key] = [val]
+            return pd.DataFrame(data)
         else:
             return np.array(instances)

{"golden_diff": "diff --git a/python/kserve/kserve/utils/utils.py b/python/kserve/kserve/utils/utils.py\n--- a/python/kserve/kserve/utils/utils.py\n+++ b/python/kserve/kserve/utils/utils.py\n@@ -16,7 +16,7 @@\n import sys\n import uuid\n from kserve.protocol.grpc.grpc_predict_v2_pb2 import InferParameter\n-from typing import Dict, Union\n+from typing import Dict, Union, List\n \n from kserve.utils.numpy_codec import from_np_dtype\n import pandas as pd\n@@ -149,6 +149,24 @@\n                 dfs.append(pd.DataFrame(input))\n             inputs = pd.concat(dfs, axis=0)\n             return inputs\n+\n+        # Handles the following input format\n+        # {'inputs': [\n+        #     [{'sepal_width_(cm)': 3.5},\n+        #      {'petal_length_(cm)': 1.4},\n+        #      {'petal_width_(cm)': 0.2},\n+        #      {'sepal_length_(cm)': 5.1}]\n+        # ]}\n+        elif isinstance(instances[0], List) and len(instances[0]) != 0 and isinstance(instances[0][0], Dict):\n+            data: Dict[str, List] = {}\n+            for instance in instances:\n+                for item in instance:\n+                    for key, val in item.items():\n+                        if key in data:\n+                            data[key].append(val)\n+                        else:\n+                            data[key] = [val]\n+            return pd.DataFrame(data)\n         else:\n             return np.array(instances)\n", "issue": "KServe lgbserver runtime error input for v1 endpoint\n/kind bug\r\n\r\n**What steps did you take and what happened:**\r\nlgbserver used to support both of following inputs formats prior to 0.11 release.\r\n\r\n```python\r\nrequest = {'inputs' : [{'sepal_width_(cm)': {0: 3.5}, \r\n'petal_length_(cm)': {0: 1.4}, \r\n'petal_width_(cm)': {0: 0.2},\r\n'sepal_length_(cm)': {0: 5.1} }]}\r\n```\r\n\r\n```python\r\nrequest2 = {'inputs': [\r\n[{'sepal_width_(cm)': 3.5}, \r\n {'petal_length_(cm)': 1.4}, \r\n {'petal_width_(cm)': 0.2}, \r\n {'sepal_length_(cm)': 5.1}]\r\n ] }\r\n```\r\n\r\nKServe only documented the first input format https://kserve.github.io/website/0.11/modelserving/v1beta1/lightgbm\r\n\r\nand in 0.11 the second input format stop working with following error\r\n\r\n```bash\r\n2023-11-03 09:06:02.099 32367 kserve ERROR [inference_error_handler():89] Exception:\r\nTraceback (most recent call last):\r\n  File \"/Users/dsun20/kserve/python/lgbserver/lgbserver/model.py\", line 62, in predict\r\n    result = self._booster.predict(instances)\r\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n  File \"/opt/homebrew/lib/python3.11/site-packages/lightgbm/basic.py\", line 4220, in predict\r\n    return predictor.predict(\r\n           ^^^^^^^^^^^^^^^^^^\r\n  File \"/opt/homebrew/lib/python3.11/site-packages/lightgbm/basic.py\", line 1047, in predict\r\n    preds, nrow = self.__pred_for_np2d(\r\n                  ^^^^^^^^^^^^^^^^^^^^^\r\n  File \"/opt/homebrew/lib/python3.11/site-packages/lightgbm/basic.py\", line 1187, in __pred_for_np2d\r\n    return self.__inner_predict_np2d(\r\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n  File \"/opt/homebrew/lib/python3.11/site-packages/lightgbm/basic.py\", line 1127, in __inner_predict_np2d\r\n    data = np.array(mat.reshape(mat.size), dtype=np.float32)\r\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\nTypeError: float() argument must be a string or a real number, not 'dict'\r\n```\r\n\r\n\r\n**What did you expect to happen:**\r\n\r\n\r\n**What's the InferenceService yaml:**\r\n[To help us debug please run `kubectl get isvc $name -n $namespace -oyaml` and paste the output]\r\n\r\n**Anything else you would like to add:**\r\n[Miscellaneous information that will assist in solving the issue.]\r\n\r\n\r\n**Environment:**\r\n\r\n- Istio Version:\r\n- Knative Version:\r\n- KServe Version:\r\n- Kubeflow version:\r\n- Cloud Environment:[k8s_istio/istio_dex/gcp_basic_auth/gcp_iap/aws/aws_cognito/ibm]\r\n- Minikube/Kind version:\r\n- Kubernetes version: (use `kubectl version`):\r\n- OS (e.g. from `/etc/os-release`):\r\n\n", "before_files": [{"content": "# Copyright 2021 The KServe Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport os\nimport sys\nimport uuid\nfrom kserve.protocol.grpc.grpc_predict_v2_pb2 import InferParameter\nfrom typing import Dict, Union\n\nfrom kserve.utils.numpy_codec import from_np_dtype\nimport pandas as pd\nimport numpy as np\nimport psutil\nfrom cloudevents.conversion import to_binary, to_structured\nfrom cloudevents.http import CloudEvent\nfrom grpc import ServicerContext\nfrom kserve.protocol.infer_type import InferOutput, InferRequest, InferResponse\n\n\ndef is_running_in_k8s():\n    return os.path.isdir('/var/run/secrets/kubernetes.io/')\n\n\ndef get_current_k8s_namespace():\n    with open('/var/run/secrets/kubernetes.io/serviceaccount/namespace', 'r') as f:\n        return f.readline()\n\n\ndef get_default_target_namespace():\n    if not is_running_in_k8s():\n        return 'default'\n    return get_current_k8s_namespace()\n\n\ndef get_isvc_namespace(inferenceservice):\n    return inferenceservice.metadata.namespace or get_default_target_namespace()\n\n\ndef get_ig_namespace(inferencegraph):\n    return inferencegraph.metadata.namespace or get_default_target_namespace()\n\n\ndef cpu_count():\n    \"\"\"Get the available CPU count for this system.\n    Takes the minimum value from the following locations:\n    - Total system cpus available on the host.\n    - CPU Affinity (if set)\n    - Cgroups limit (if set)\n    \"\"\"\n    count = os.cpu_count()\n\n    # Check CPU affinity if available\n    try:\n        affinity_count = len(psutil.Process().cpu_affinity())\n        if affinity_count > 0:\n            count = min(count, affinity_count)\n    except Exception:\n        pass\n\n    # Check cgroups if available\n    if sys.platform == \"linux\":\n        try:\n            with open(\"/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us\") as f:\n                quota = int(f.read())\n            with open(\"/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_period_us\") as f:\n                period = int(f.read())\n            cgroups_count = int(quota / period)\n            if cgroups_count > 0:\n                count = min(count, cgroups_count)\n        except Exception:\n            pass\n\n    return count\n\n\ndef is_structured_cloudevent(body: Dict) -> bool:\n    \"\"\"Returns True if the JSON request body resembles a structured CloudEvent\"\"\"\n    return \"time\" in body \\\n           and \"type\" in body \\\n           and \"source\" in body \\\n           and \"id\" in body \\\n           and \"specversion\" in body \\\n           and \"data\" in body\n\n\ndef create_response_cloudevent(model_name: str, response: Dict, req_attributes: Dict,\n                               binary_event=False) -> tuple:\n    ce_attributes = {}\n\n    if os.getenv(\"CE_MERGE\", \"false\").lower() == \"true\":\n        if binary_event:\n            ce_attributes = req_attributes\n            if \"datacontenttype\" in ce_attributes:  # Optional field so must check\n                del ce_attributes[\"datacontenttype\"]\n        else:\n            ce_attributes = req_attributes\n\n        # Remove these fields so we generate new ones\n        del ce_attributes[\"id\"]\n        del ce_attributes[\"time\"]\n\n    ce_attributes[\"type\"] = os.getenv(\"CE_TYPE\", \"io.kserve.inference.response\")\n    ce_attributes[\"source\"] = os.getenv(\"CE_SOURCE\", f\"io.kserve.inference.{model_name}\")\n\n    event = CloudEvent(ce_attributes, response)\n\n    if binary_event:\n        event_headers, event_body = to_binary(event)\n    else:\n        event_headers, event_body = to_structured(event)\n\n    return event_headers, event_body\n\n\ndef generate_uuid() -> str:\n    return str(uuid.uuid4())\n\n\ndef to_headers(context: ServicerContext) -> Dict[str, str]:\n    metadata = context.invocation_metadata()\n    if hasattr(context, \"trailing_metadata\"):\n        metadata += context.trailing_metadata()\n    headers = {}\n    for metadatum in metadata:\n        headers[metadatum.key] = metadatum.value\n\n    return headers\n\n\ndef get_predict_input(payload: Union[Dict, InferRequest]) -> Union[np.ndarray, pd.DataFrame]:\n    if isinstance(payload, Dict):\n        instances = payload[\"inputs\"] if \"inputs\" in payload else payload[\"instances\"]\n        if len(instances) == 0:\n            return np.array(instances)\n        if isinstance(instances[0], Dict):\n            dfs = []\n            for input in instances:\n                dfs.append(pd.DataFrame(input))\n            inputs = pd.concat(dfs, axis=0)\n            return inputs\n        else:\n            return np.array(instances)\n\n    elif isinstance(payload, InferRequest):\n        content_type = ''\n        parameters = payload.parameters\n        if parameters:\n            if isinstance(parameters.get(\"content_type\"), InferParameter):\n                # for v2 grpc, we get InferParameter obj eg: {\"content_type\": string_param: \"pd\"}\n                content_type = str(parameters.get(\"content_type\").string_param)\n            else:\n                # for v2 http, we get string eg: {\"content_type\": \"pd\"}\n                content_type = parameters.get(\"content_type\")\n\n        if content_type == \"pd\":\n            return payload.as_dataframe()\n        else:\n            input = payload.inputs[0]\n            return input.as_numpy()\n\n\ndef get_predict_response(payload: Union[Dict, InferRequest], result: Union[np.ndarray, pd.DataFrame],\n                         model_name: str) -> Union[Dict, InferResponse]:\n    if isinstance(payload, Dict):\n        infer_outputs = result\n        if isinstance(result, pd.DataFrame):\n            infer_outputs = []\n            for label, row in result.iterrows():\n                infer_outputs.append(row.to_dict())\n        elif isinstance(result, np.ndarray):\n            infer_outputs = result.tolist()\n        return {\"predictions\": infer_outputs}\n    elif isinstance(payload, InferRequest):\n        infer_outputs = []\n        if isinstance(result, pd.DataFrame):\n            for col in result.columns:\n                infer_output = InferOutput(\n                    name=col,\n                    shape=list(result[col].shape),\n                    datatype=from_np_dtype(result[col].dtype),\n                    data=result[col].tolist()\n                )\n                infer_outputs.append(infer_output)\n        else:\n            infer_output = InferOutput(\n                name=\"output-0\",\n                shape=list(result.shape),\n                datatype=from_np_dtype(result.dtype),\n                data=result.flatten().tolist()\n            )\n            infer_outputs.append(infer_output)\n        return InferResponse(\n            model_name=model_name,\n            infer_outputs=infer_outputs,\n            response_id=payload.id if payload.id else generate_uuid()\n        )\n\n\ndef strtobool(val: str) -> bool:\n    \"\"\"Convert a string representation of truth to True or False.\n\n    True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values\n    are 'n', 'no', 'f', 'false', 'off', and '0'.  Raises ValueError if\n    'val' is anything else.\n\n    Adapted from deprecated `distutils`\n    https://github.com/python/cpython/blob/3.11/Lib/distutils/util.py\n    \"\"\"\n    val = val.lower()\n    if val in ('y', 'yes', 't', 'true', 'on', '1'):\n        return True\n    elif val in ('n', 'no', 'f', 'false', 'off', '0'):\n        return False\n    else:\n        raise ValueError(\"invalid truth value %r\" % (val,))\n", "path": "python/kserve/kserve/utils/utils.py"}], "after_files": [{"content": "# Copyright 2021 The KServe Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport os\nimport sys\nimport uuid\nfrom kserve.protocol.grpc.grpc_predict_v2_pb2 import InferParameter\nfrom typing import Dict, Union, List\n\nfrom kserve.utils.numpy_codec import from_np_dtype\nimport pandas as pd\nimport numpy as np\nimport psutil\nfrom cloudevents.conversion import to_binary, to_structured\nfrom cloudevents.http import CloudEvent\nfrom grpc import ServicerContext\nfrom kserve.protocol.infer_type import InferOutput, InferRequest, InferResponse\n\n\ndef is_running_in_k8s():\n    return os.path.isdir('/var/run/secrets/kubernetes.io/')\n\n\ndef get_current_k8s_namespace():\n    with open('/var/run/secrets/kubernetes.io/serviceaccount/namespace', 'r') as f:\n        return f.readline()\n\n\ndef get_default_target_namespace():\n    if not is_running_in_k8s():\n        return 'default'\n    return get_current_k8s_namespace()\n\n\ndef get_isvc_namespace(inferenceservice):\n    return inferenceservice.metadata.namespace or get_default_target_namespace()\n\n\ndef get_ig_namespace(inferencegraph):\n    return inferencegraph.metadata.namespace or get_default_target_namespace()\n\n\ndef cpu_count():\n    \"\"\"Get the available CPU count for this system.\n    Takes the minimum value from the following locations:\n    - Total system cpus available on the host.\n    - CPU Affinity (if set)\n    - Cgroups limit (if set)\n    \"\"\"\n    count = os.cpu_count()\n\n    # Check CPU affinity if available\n    try:\n        affinity_count = len(psutil.Process().cpu_affinity())\n        if affinity_count > 0:\n            count = min(count, affinity_count)\n    except Exception:\n        pass\n\n    # Check cgroups if available\n    if sys.platform == \"linux\":\n        try:\n            with open(\"/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us\") as f:\n                quota = int(f.read())\n            with open(\"/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_period_us\") as f:\n                period = int(f.read())\n            cgroups_count = int(quota / period)\n            if cgroups_count > 0:\n                count = min(count, cgroups_count)\n        except Exception:\n            pass\n\n    return count\n\n\ndef is_structured_cloudevent(body: Dict) -> bool:\n    \"\"\"Returns True if the JSON request body resembles a structured CloudEvent\"\"\"\n    return \"time\" in body \\\n           and \"type\" in body \\\n           and \"source\" in body \\\n           and \"id\" in body \\\n           and \"specversion\" in body \\\n           and \"data\" in body\n\n\ndef create_response_cloudevent(model_name: str, response: Dict, req_attributes: Dict,\n                               binary_event=False) -> tuple:\n    ce_attributes = {}\n\n    if os.getenv(\"CE_MERGE\", \"false\").lower() == \"true\":\n        if binary_event:\n            ce_attributes = req_attributes\n            if \"datacontenttype\" in ce_attributes:  # Optional field so must check\n                del ce_attributes[\"datacontenttype\"]\n        else:\n            ce_attributes = req_attributes\n\n        # Remove these fields so we generate new ones\n        del ce_attributes[\"id\"]\n        del ce_attributes[\"time\"]\n\n    ce_attributes[\"type\"] = os.getenv(\"CE_TYPE\", \"io.kserve.inference.response\")\n    ce_attributes[\"source\"] = os.getenv(\"CE_SOURCE\", f\"io.kserve.inference.{model_name}\")\n\n    event = CloudEvent(ce_attributes, response)\n\n    if binary_event:\n        event_headers, event_body = to_binary(event)\n    else:\n        event_headers, event_body = to_structured(event)\n\n    return event_headers, event_body\n\n\ndef generate_uuid() -> str:\n    return str(uuid.uuid4())\n\n\ndef to_headers(context: ServicerContext) -> Dict[str, str]:\n    metadata = context.invocation_metadata()\n    if hasattr(context, \"trailing_metadata\"):\n        metadata += context.trailing_metadata()\n    headers = {}\n    for metadatum in metadata:\n        headers[metadatum.key] = metadatum.value\n\n    return headers\n\n\ndef get_predict_input(payload: Union[Dict, InferRequest]) -> Union[np.ndarray, pd.DataFrame]:\n    if isinstance(payload, Dict):\n        instances = payload[\"inputs\"] if \"inputs\" in payload else payload[\"instances\"]\n        if len(instances) == 0:\n            return np.array(instances)\n        if isinstance(instances[0], Dict):\n            dfs = []\n            for input in instances:\n                dfs.append(pd.DataFrame(input))\n            inputs = pd.concat(dfs, axis=0)\n            return inputs\n\n        # Handles the following input format\n        # {'inputs': [\n        #     [{'sepal_width_(cm)': 3.5},\n        #      {'petal_length_(cm)': 1.4},\n        #      {'petal_width_(cm)': 0.2},\n        #      {'sepal_length_(cm)': 5.1}]\n        # ]}\n        elif isinstance(instances[0], List) and len(instances[0]) != 0 and isinstance(instances[0][0], Dict):\n            data: Dict[str, List] = {}\n            for instance in instances:\n                for item in instance:\n                    for key, val in item.items():\n                        if key in data:\n                            data[key].append(val)\n                        else:\n                            data[key] = [val]\n            return pd.DataFrame(data)\n        else:\n            return np.array(instances)\n\n    elif isinstance(payload, InferRequest):\n        content_type = ''\n        parameters = payload.parameters\n        if parameters:\n            if isinstance(parameters.get(\"content_type\"), InferParameter):\n                # for v2 grpc, we get InferParameter obj eg: {\"content_type\": string_param: \"pd\"}\n                content_type = str(parameters.get(\"content_type\").string_param)\n            else:\n                # for v2 http, we get string eg: {\"content_type\": \"pd\"}\n                content_type = parameters.get(\"content_type\")\n\n        if content_type == \"pd\":\n            return payload.as_dataframe()\n        else:\n            input = payload.inputs[0]\n            return input.as_numpy()\n\n\ndef get_predict_response(payload: Union[Dict, InferRequest], result: Union[np.ndarray, pd.DataFrame],\n                         model_name: str) -> Union[Dict, InferResponse]:\n    if isinstance(payload, Dict):\n        infer_outputs = result\n        if isinstance(result, pd.DataFrame):\n            infer_outputs = []\n            for label, row in result.iterrows():\n                infer_outputs.append(row.to_dict())\n        elif isinstance(result, np.ndarray):\n            infer_outputs = result.tolist()\n        return {\"predictions\": infer_outputs}\n    elif isinstance(payload, InferRequest):\n        infer_outputs = []\n        if isinstance(result, pd.DataFrame):\n            for col in result.columns:\n                infer_output = InferOutput(\n                    name=col,\n                    shape=list(result[col].shape),\n                    datatype=from_np_dtype(result[col].dtype),\n                    data=result[col].tolist()\n                )\n                infer_outputs.append(infer_output)\n        else:\n            infer_output = InferOutput(\n                name=\"output-0\",\n                shape=list(result.shape),\n                datatype=from_np_dtype(result.dtype),\n                data=result.flatten().tolist()\n            )\n            infer_outputs.append(infer_output)\n        return InferResponse(\n            model_name=model_name,\n            infer_outputs=infer_outputs,\n            response_id=payload.id if payload.id else generate_uuid()\n        )\n\n\ndef strtobool(val: str) -> bool:\n    \"\"\"Convert a string representation of truth to True or False.\n\n    True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values\n    are 'n', 'no', 'f', 'false', 'off', and '0'.  Raises ValueError if\n    'val' is anything else.\n\n    Adapted from deprecated `distutils`\n    https://github.com/python/cpython/blob/3.11/Lib/distutils/util.py\n    \"\"\"\n    val = val.lower()\n    if val in ('y', 'yes', 't', 'true', 'on', '1'):\n        return True\n    elif val in ('n', 'no', 'f', 'false', 'off', '0'):\n        return False\n    else:\n        raise ValueError(\"invalid truth value %r\" % (val,))\n", "path": "python/kserve/kserve/utils/utils.py"}]}