We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Support for Array type hints in APIs that take Python native functions (e.g., DataFrame.apply)
```python
import databricks.koalas as ks

def tokenizeDF(col1) -> ks.Series[np.array(... ???)]:
    pass
```

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `databricks/koalas/typedef/typehints.py`
Content:
```
1 #
2 # Copyright (C) 2019 Databricks, Inc.
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 #     http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15 #
16 
17 """
18 Utilities to deal with types. This is mostly focused on python3.
19 """
20 import typing
21 import datetime
22 import decimal
23 from inspect import getfullargspec, isclass
24 
25 import numpy as np
26 import pandas as pd
27 from pandas.api.types import is_datetime64_dtype, is_datetime64tz_dtype
28 import pyarrow as pa
29 import pyspark.sql.types as types
30 
31 try:
32     from pyspark.sql.types import to_arrow_type, from_arrow_type
33 except ImportError:
34     from pyspark.sql.pandas.types import to_arrow_type, from_arrow_type
35 
36 from databricks import koalas as ks  # For running doctests and reference resolution in PyCharm.
37 from databricks.koalas.typedef.string_typehints import resolve_string_type_hint
38 
39 T = typing.TypeVar("T")
40 
41 Scalar = typing.Union[
42     int, float, bool, str, bytes, decimal.Decimal, datetime.date, datetime.datetime, None
43 ]
44 
45 
46 # A column of data, with the data type.
47 class SeriesType(typing.Generic[T]):
48     def __init__(self, tpe):
49         self.tpe = tpe  # type: types.DataType
50 
51     def __repr__(self):
52         return "SeriesType[{}]".format(self.tpe)
53 
54 
55 class DataFrameType(object):
56     def __init__(self, tpe, names=None):
57         if names is None:
58             # Default names `c0, c1, ... cn`.
59             self.tpe = types.StructType(
60                 [types.StructField("c%s" % i, tpe[i]) for i in range(len(tpe))]
61             )  # type: types.StructType
62         else:
63             self.tpe = types.StructType(
64                 [types.StructField(n, t) for n, t in zip(names, tpe)]
65             )  # type: types.StructType
66 
67     def __repr__(self):
68         return "DataFrameType[{}]".format(self.tpe)
69 
70 
71 # The type is a scalar type that is furthermore understood by Spark.
72 class ScalarType(object):
73     def __init__(self, tpe):
74         self.tpe = tpe  # type: types.DataType
75 
76     def __repr__(self):
77         return "ScalarType[{}]".format(self.tpe)
78 
79 
80 # The type is left unspecified or we do not know about this type.
81 class UnknownType(object):
82     def __init__(self, tpe):
83         self.tpe = tpe
84 
85     def __repr__(self):
86         return "UnknownType[{}]".format(self.tpe)
87 
88 
89 class NameTypeHolder(object):
90     name = None
91     tpe = None
92 
93 
94 def as_spark_type(tpe) -> types.DataType:
95     """
96     Given a Python type, returns the equivalent spark type.
97     Accepts:
98     - the built-in types in Python
99     - the built-in types in numpy
100     - list of pairs of (field_name, type)
101     - dictionaries of field_name -> type
102     - Python3's typing system
103     """
104     # TODO: Add "boolean" and "string" types.
105     # ArrayType
106     if tpe in (np.ndarray,):
107         # TODO: support other child types
108         return types.ArrayType(types.StringType())
109     # BinaryType
110     elif tpe in (bytes, np.character, np.bytes_, np.string_):
111         return types.BinaryType()
112     # BooleanType
113     elif tpe in (bool, np.bool, "bool", "?"):
114         return types.BooleanType()
115     # DateType
116     elif tpe in (datetime.date,):
117         return types.DateType()
118     # NumericType
119     elif tpe in (np.int8, np.byte, "int8", "byte", "b"):
120         return types.ByteType()
121     elif tpe in (decimal.Decimal,):
122         # TODO: considering about the precision & scale for decimal type.
123         return types.DecimalType(38, 18)
124     elif tpe in (float, np.float, np.float64, "float", "float64", "double"):
125         return types.DoubleType()
126     elif tpe in (np.float32, "float32", "f"):
127         return types.FloatType()
128     elif tpe in (np.int32, "int32", "i"):
129         return types.IntegerType()
130     elif tpe in (int, np.int, np.int64, "int", "int64", "long", "bigint"):
131         return types.LongType()
132     elif tpe in (np.int16, "int16", "short"):
133         return types.ShortType()
134     # StringType
135     elif tpe in (str, np.unicode_, "str", "U"):
136         return types.StringType()
137     # TimestampType
138     elif tpe in (datetime.datetime, np.datetime64, "datetime64[ns]", "M"):
139         return types.TimestampType()
140     else:
141         raise TypeError("Type %s was not understood." % tpe)
142 
143 
144 def spark_type_to_pandas_dtype(spark_type):
145     """ Return the given Spark DataType to pandas dtype. """
146     if isinstance(spark_type, (types.DateType, types.StructType, types.UserDefinedType)):
147         return np.dtype("object")
148     elif isinstance(spark_type, types.TimestampType):
149         return np.dtype("datetime64[ns]")
150     else:
151         return np.dtype(to_arrow_type(spark_type).to_pandas_dtype())
152 
153 
154 def infer_pd_series_spark_type(s: pd.Series) -> types.DataType:
155     """Infer Spark DataType from pandas Series dtype.
156 
157     :param s: :class:`pandas.Series` to be inferred
158     :return: the inferred Spark data type
159     """
160     dt = s.dtype
161     if dt == np.dtype("object"):
162         if len(s) == 0 or s.isnull().all():
163             raise ValueError("can not infer schema from empty or null dataset")
164         elif hasattr(s[0], "__UDT__"):
165             return s[0].__UDT__
166         else:
167             return from_arrow_type(pa.Array.from_pandas(s).type)
168     elif is_datetime64_dtype(dt) or is_datetime64tz_dtype(dt):
169         return types.TimestampType()
170     else:
171         return from_arrow_type(pa.from_numpy_dtype(dt))
172 
173 
174 def infer_return_type(f) -> typing.Union[SeriesType, DataFrameType, ScalarType, UnknownType]:
175     """
176     >>> def func() -> int:
177     ...    pass
178     >>> infer_return_type(func).tpe
179     LongType
180 
181     >>> def func() -> ks.Series[int]:
182     ...    pass
183     >>> infer_return_type(func).tpe
184     LongType
185 
186     >>> def func() -> ks.DataFrame[np.float, str]:
187     ...    pass
188     >>> infer_return_type(func).tpe
189     StructType(List(StructField(c0,DoubleType,true),StructField(c1,StringType,true)))
190 
191     >>> def func() -> ks.DataFrame[np.float]:
192     ...    pass
193     >>> infer_return_type(func).tpe
194     StructType(List(StructField(c0,DoubleType,true)))
195 
196     >>> def func() -> 'int':
197     ...    pass
198     >>> infer_return_type(func).tpe
199     LongType
200 
201     >>> def func() -> 'ks.Series[int]':
202     ...    pass
203     >>> infer_return_type(func).tpe
204     LongType
205 
206     >>> def func() -> 'ks.DataFrame[np.float, str]':
207     ...    pass
208     >>> infer_return_type(func).tpe
209     StructType(List(StructField(c0,DoubleType,true),StructField(c1,StringType,true)))
210 
211     >>> def func() -> 'ks.DataFrame[np.float]':
212     ...    pass
213     >>> infer_return_type(func).tpe
214     StructType(List(StructField(c0,DoubleType,true)))
215 
216     >>> def func() -> ks.DataFrame['a': np.float, 'b': int]:
217     ...     pass
218     >>> infer_return_type(func).tpe
219     StructType(List(StructField(a,DoubleType,true),StructField(b,LongType,true)))
220 
221     >>> def func() -> "ks.DataFrame['a': np.float, 'b': int]":
222     ...     pass
223     >>> infer_return_type(func).tpe
224     StructType(List(StructField(a,DoubleType,true),StructField(b,LongType,true)))
225 
226     >>> pdf = pd.DataFrame({"a": [1, 2, 3], "b": [3, 4, 5]})
227     >>> def func() -> ks.DataFrame[pdf.dtypes]:
228     ...     pass
229     >>> infer_return_type(func).tpe
230     StructType(List(StructField(c0,LongType,true),StructField(c1,LongType,true)))
231 
232     >>> pdf = pd.DataFrame({"a": [1, 2, 3], "b": [3, 4, 5]})
233     >>> def func() -> ks.DataFrame[zip(pdf.columns, pdf.dtypes)]:
234     ...     pass
235     >>> infer_return_type(func).tpe
236     StructType(List(StructField(a,LongType,true),StructField(b,LongType,true)))
237     """
238     # We should re-import to make sure the class 'SeriesType' is not treated as a class
239     # within this module locally. See Series.__class_getitem__ which imports this class
240     # canonically.
241     from databricks.koalas.typedef import SeriesType, NameTypeHolder
242 
243     spec = getfullargspec(f)
244     tpe = spec.annotations.get("return", None)
245     if isinstance(tpe, str):
246         # This type hint can happen when given hints are string to avoid forward reference.
247         tpe = resolve_string_type_hint(tpe)
248     if hasattr(tpe, "__origin__") and (
249         issubclass(tpe.__origin__, SeriesType) or tpe.__origin__ == ks.Series
250     ):
251         # TODO: remove "tpe.__origin__ == ks.Series" when we drop Python 3.5 and 3.6.
252         inner = as_spark_type(tpe.__args__[0])
253         return SeriesType(inner)
254 
255     if hasattr(tpe, "__origin__") and tpe.__origin__ == ks.DataFrame:
256         # When Python version is lower then 3.7. Unwrap it to a Tuple type
257         # hints.
258         tpe = tpe.__args__[0]
259 
260     # Note that, DataFrame type hints will create a Tuple.
261     # Python 3.6 has `__name__`. Python 3.7 and 3.8 have `_name`.
262     # Check if the name is Tuple.
263     name = getattr(tpe, "_name", getattr(tpe, "__name__", None))
264     if name == "Tuple":
265         tuple_type = tpe
266         if hasattr(tuple_type, "__tuple_params__"):
267             # Python 3.5.0 to 3.5.2 has '__tuple_params__' instead.
268             # See https://github.com/python/cpython/blob/v3.5.2/Lib/typing.py
269             parameters = getattr(tuple_type, "__tuple_params__")
270         else:
271             parameters = getattr(tuple_type, "__args__")
272         if len(parameters) > 0 and all(
273             isclass(p) and issubclass(p, NameTypeHolder) for p in parameters
274         ):
275             names = [p.name for p in parameters if issubclass(p, NameTypeHolder)]
276             types = [p.tpe for p in parameters if issubclass(p, NameTypeHolder)]
277             return DataFrameType([as_spark_type(t) for t in types], names)
278         return DataFrameType([as_spark_type(t) for t in parameters])
279     inner = as_spark_type(tpe)
280     if inner is None:
281         return UnknownType(tpe)
282     else:
283         return ScalarType(inner)
284
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/databricks/koalas/typedef/typehints.py b/databricks/koalas/typedef/typehints.py
--- a/databricks/koalas/typedef/typehints.py
+++ b/databricks/koalas/typedef/typehints.py
@@ -104,8 +104,9 @@
     # TODO: Add "boolean" and "string" types.
     # ArrayType
     if tpe in (np.ndarray,):
-        # TODO: support other child types
         return types.ArrayType(types.StringType())
+    elif hasattr(tpe, "__origin__") and issubclass(tpe.__origin__, list):
+        return types.ArrayType(as_spark_type(tpe.__args__[0]))
     # BinaryType
     elif tpe in (bytes, np.character, np.bytes_, np.string_):
         return types.BinaryType()

{"golden_diff": "diff --git a/databricks/koalas/typedef/typehints.py b/databricks/koalas/typedef/typehints.py\n--- a/databricks/koalas/typedef/typehints.py\n+++ b/databricks/koalas/typedef/typehints.py\n@@ -104,8 +104,9 @@\n     # TODO: Add \"boolean\" and \"string\" types.\n     # ArrayType\n     if tpe in (np.ndarray,):\n-        # TODO: support other child types\n         return types.ArrayType(types.StringType())\n+    elif hasattr(tpe, \"__origin__\") and issubclass(tpe.__origin__, list):\n+        return types.ArrayType(as_spark_type(tpe.__args__[0]))\n     # BinaryType\n     elif tpe in (bytes, np.character, np.bytes_, np.string_):\n         return types.BinaryType()\n", "issue": "Support for Array type hints in APIs that take Python native functions (e.g., DataFrame.apply)\n```python\r\nimport databricks.koalas as ks\r\n\r\ndef tokenizeDF(col1) -> ks.Series[np.array(... ???)]:\r\n    pass\r\n```\n", "before_files": [{"content": "#\n# Copyright (C) 2019 Databricks, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\"\"\"\nUtilities to deal with types. This is mostly focused on python3.\n\"\"\"\nimport typing\nimport datetime\nimport decimal\nfrom inspect import getfullargspec, isclass\n\nimport numpy as np\nimport pandas as pd\nfrom pandas.api.types import is_datetime64_dtype, is_datetime64tz_dtype\nimport pyarrow as pa\nimport pyspark.sql.types as types\n\ntry:\n    from pyspark.sql.types import to_arrow_type, from_arrow_type\nexcept ImportError:\n    from pyspark.sql.pandas.types import to_arrow_type, from_arrow_type\n\nfrom databricks import koalas as ks  # For running doctests and reference resolution in PyCharm.\nfrom databricks.koalas.typedef.string_typehints import resolve_string_type_hint\n\nT = typing.TypeVar(\"T\")\n\nScalar = typing.Union[\n    int, float, bool, str, bytes, decimal.Decimal, datetime.date, datetime.datetime, None\n]\n\n\n# A column of data, with the data type.\nclass SeriesType(typing.Generic[T]):\n    def __init__(self, tpe):\n        self.tpe = tpe  # type: types.DataType\n\n    def __repr__(self):\n        return \"SeriesType[{}]\".format(self.tpe)\n\n\nclass DataFrameType(object):\n    def __init__(self, tpe, names=None):\n        if names is None:\n            # Default names `c0, c1, ... cn`.\n            self.tpe = types.StructType(\n                [types.StructField(\"c%s\" % i, tpe[i]) for i in range(len(tpe))]\n            )  # type: types.StructType\n        else:\n            self.tpe = types.StructType(\n                [types.StructField(n, t) for n, t in zip(names, tpe)]\n            )  # type: types.StructType\n\n    def __repr__(self):\n        return \"DataFrameType[{}]\".format(self.tpe)\n\n\n# The type is a scalar type that is furthermore understood by Spark.\nclass ScalarType(object):\n    def __init__(self, tpe):\n        self.tpe = tpe  # type: types.DataType\n\n    def __repr__(self):\n        return \"ScalarType[{}]\".format(self.tpe)\n\n\n# The type is left unspecified or we do not know about this type.\nclass UnknownType(object):\n    def __init__(self, tpe):\n        self.tpe = tpe\n\n    def __repr__(self):\n        return \"UnknownType[{}]\".format(self.tpe)\n\n\nclass NameTypeHolder(object):\n    name = None\n    tpe = None\n\n\ndef as_spark_type(tpe) -> types.DataType:\n    \"\"\"\n    Given a Python type, returns the equivalent spark type.\n    Accepts:\n    - the built-in types in Python\n    - the built-in types in numpy\n    - list of pairs of (field_name, type)\n    - dictionaries of field_name -> type\n    - Python3's typing system\n    \"\"\"\n    # TODO: Add \"boolean\" and \"string\" types.\n    # ArrayType\n    if tpe in (np.ndarray,):\n        # TODO: support other child types\n        return types.ArrayType(types.StringType())\n    # BinaryType\n    elif tpe in (bytes, np.character, np.bytes_, np.string_):\n        return types.BinaryType()\n    # BooleanType\n    elif tpe in (bool, np.bool, \"bool\", \"?\"):\n        return types.BooleanType()\n    # DateType\n    elif tpe in (datetime.date,):\n        return types.DateType()\n    # NumericType\n    elif tpe in (np.int8, np.byte, \"int8\", \"byte\", \"b\"):\n        return types.ByteType()\n    elif tpe in (decimal.Decimal,):\n        # TODO: considering about the precision & scale for decimal type.\n        return types.DecimalType(38, 18)\n    elif tpe in (float, np.float, np.float64, \"float\", \"float64\", \"double\"):\n        return types.DoubleType()\n    elif tpe in (np.float32, \"float32\", \"f\"):\n        return types.FloatType()\n    elif tpe in (np.int32, \"int32\", \"i\"):\n        return types.IntegerType()\n    elif tpe in (int, np.int, np.int64, \"int\", \"int64\", \"long\", \"bigint\"):\n        return types.LongType()\n    elif tpe in (np.int16, \"int16\", \"short\"):\n        return types.ShortType()\n    # StringType\n    elif tpe in (str, np.unicode_, \"str\", \"U\"):\n        return types.StringType()\n    # TimestampType\n    elif tpe in (datetime.datetime, np.datetime64, \"datetime64[ns]\", \"M\"):\n        return types.TimestampType()\n    else:\n        raise TypeError(\"Type %s was not understood.\" % tpe)\n\n\ndef spark_type_to_pandas_dtype(spark_type):\n    \"\"\" Return the given Spark DataType to pandas dtype. \"\"\"\n    if isinstance(spark_type, (types.DateType, types.StructType, types.UserDefinedType)):\n        return np.dtype(\"object\")\n    elif isinstance(spark_type, types.TimestampType):\n        return np.dtype(\"datetime64[ns]\")\n    else:\n        return np.dtype(to_arrow_type(spark_type).to_pandas_dtype())\n\n\ndef infer_pd_series_spark_type(s: pd.Series) -> types.DataType:\n    \"\"\"Infer Spark DataType from pandas Series dtype.\n\n    :param s: :class:`pandas.Series` to be inferred\n    :return: the inferred Spark data type\n    \"\"\"\n    dt = s.dtype\n    if dt == np.dtype(\"object\"):\n        if len(s) == 0 or s.isnull().all():\n            raise ValueError(\"can not infer schema from empty or null dataset\")\n        elif hasattr(s[0], \"__UDT__\"):\n            return s[0].__UDT__\n        else:\n            return from_arrow_type(pa.Array.from_pandas(s).type)\n    elif is_datetime64_dtype(dt) or is_datetime64tz_dtype(dt):\n        return types.TimestampType()\n    else:\n        return from_arrow_type(pa.from_numpy_dtype(dt))\n\n\ndef infer_return_type(f) -> typing.Union[SeriesType, DataFrameType, ScalarType, UnknownType]:\n    \"\"\"\n    >>> def func() -> int:\n    ...    pass\n    >>> infer_return_type(func).tpe\n    LongType\n\n    >>> def func() -> ks.Series[int]:\n    ...    pass\n    >>> infer_return_type(func).tpe\n    LongType\n\n    >>> def func() -> ks.DataFrame[np.float, str]:\n    ...    pass\n    >>> infer_return_type(func).tpe\n    StructType(List(StructField(c0,DoubleType,true),StructField(c1,StringType,true)))\n\n    >>> def func() -> ks.DataFrame[np.float]:\n    ...    pass\n    >>> infer_return_type(func).tpe\n    StructType(List(StructField(c0,DoubleType,true)))\n\n    >>> def func() -> 'int':\n    ...    pass\n    >>> infer_return_type(func).tpe\n    LongType\n\n    >>> def func() -> 'ks.Series[int]':\n    ...    pass\n    >>> infer_return_type(func).tpe\n    LongType\n\n    >>> def func() -> 'ks.DataFrame[np.float, str]':\n    ...    pass\n    >>> infer_return_type(func).tpe\n    StructType(List(StructField(c0,DoubleType,true),StructField(c1,StringType,true)))\n\n    >>> def func() -> 'ks.DataFrame[np.float]':\n    ...    pass\n    >>> infer_return_type(func).tpe\n    StructType(List(StructField(c0,DoubleType,true)))\n\n    >>> def func() -> ks.DataFrame['a': np.float, 'b': int]:\n    ...     pass\n    >>> infer_return_type(func).tpe\n    StructType(List(StructField(a,DoubleType,true),StructField(b,LongType,true)))\n\n    >>> def func() -> \"ks.DataFrame['a': np.float, 'b': int]\":\n    ...     pass\n    >>> infer_return_type(func).tpe\n    StructType(List(StructField(a,DoubleType,true),StructField(b,LongType,true)))\n\n    >>> pdf = pd.DataFrame({\"a\": [1, 2, 3], \"b\": [3, 4, 5]})\n    >>> def func() -> ks.DataFrame[pdf.dtypes]:\n    ...     pass\n    >>> infer_return_type(func).tpe\n    StructType(List(StructField(c0,LongType,true),StructField(c1,LongType,true)))\n\n    >>> pdf = pd.DataFrame({\"a\": [1, 2, 3], \"b\": [3, 4, 5]})\n    >>> def func() -> ks.DataFrame[zip(pdf.columns, pdf.dtypes)]:\n    ...     pass\n    >>> infer_return_type(func).tpe\n    StructType(List(StructField(a,LongType,true),StructField(b,LongType,true)))\n    \"\"\"\n    # We should re-import to make sure the class 'SeriesType' is not treated as a class\n    # within this module locally. See Series.__class_getitem__ which imports this class\n    # canonically.\n    from databricks.koalas.typedef import SeriesType, NameTypeHolder\n\n    spec = getfullargspec(f)\n    tpe = spec.annotations.get(\"return\", None)\n    if isinstance(tpe, str):\n        # This type hint can happen when given hints are string to avoid forward reference.\n        tpe = resolve_string_type_hint(tpe)\n    if hasattr(tpe, \"__origin__\") and (\n        issubclass(tpe.__origin__, SeriesType) or tpe.__origin__ == ks.Series\n    ):\n        # TODO: remove \"tpe.__origin__ == ks.Series\" when we drop Python 3.5 and 3.6.\n        inner = as_spark_type(tpe.__args__[0])\n        return SeriesType(inner)\n\n    if hasattr(tpe, \"__origin__\") and tpe.__origin__ == ks.DataFrame:\n        # When Python version is lower then 3.7. Unwrap it to a Tuple type\n        # hints.\n        tpe = tpe.__args__[0]\n\n    # Note that, DataFrame type hints will create a Tuple.\n    # Python 3.6 has `__name__`. Python 3.7 and 3.8 have `_name`.\n    # Check if the name is Tuple.\n    name = getattr(tpe, \"_name\", getattr(tpe, \"__name__\", None))\n    if name == \"Tuple\":\n        tuple_type = tpe\n        if hasattr(tuple_type, \"__tuple_params__\"):\n            # Python 3.5.0 to 3.5.2 has '__tuple_params__' instead.\n            # See https://github.com/python/cpython/blob/v3.5.2/Lib/typing.py\n            parameters = getattr(tuple_type, \"__tuple_params__\")\n        else:\n            parameters = getattr(tuple_type, \"__args__\")\n        if len(parameters) > 0 and all(\n            isclass(p) and issubclass(p, NameTypeHolder) for p in parameters\n        ):\n            names = [p.name for p in parameters if issubclass(p, NameTypeHolder)]\n            types = [p.tpe for p in parameters if issubclass(p, NameTypeHolder)]\n            return DataFrameType([as_spark_type(t) for t in types], names)\n        return DataFrameType([as_spark_type(t) for t in parameters])\n    inner = as_spark_type(tpe)\n    if inner is None:\n        return UnknownType(tpe)\n    else:\n        return ScalarType(inner)\n", "path": "databricks/koalas/typedef/typehints.py"}], "after_files": [{"content": "#\n# Copyright (C) 2019 Databricks, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\"\"\"\nUtilities to deal with types. This is mostly focused on python3.\n\"\"\"\nimport typing\nimport datetime\nimport decimal\nfrom inspect import getfullargspec, isclass\n\nimport numpy as np\nimport pandas as pd\nfrom pandas.api.types import is_datetime64_dtype, is_datetime64tz_dtype\nimport pyarrow as pa\nimport pyspark.sql.types as types\n\ntry:\n    from pyspark.sql.types import to_arrow_type, from_arrow_type\nexcept ImportError:\n    from pyspark.sql.pandas.types import to_arrow_type, from_arrow_type\n\nfrom databricks import koalas as ks  # For running doctests and reference resolution in PyCharm.\nfrom databricks.koalas.typedef.string_typehints import resolve_string_type_hint\n\nT = typing.TypeVar(\"T\")\n\nScalar = typing.Union[\n    int, float, bool, str, bytes, decimal.Decimal, datetime.date, datetime.datetime, None\n]\n\n\n# A column of data, with the data type.\nclass SeriesType(typing.Generic[T]):\n    def __init__(self, tpe):\n        self.tpe = tpe  # type: types.DataType\n\n    def __repr__(self):\n        return \"SeriesType[{}]\".format(self.tpe)\n\n\nclass DataFrameType(object):\n    def __init__(self, tpe, names=None):\n        if names is None:\n            # Default names `c0, c1, ... cn`.\n            self.tpe = types.StructType(\n                [types.StructField(\"c%s\" % i, tpe[i]) for i in range(len(tpe))]\n            )  # type: types.StructType\n        else:\n            self.tpe = types.StructType(\n                [types.StructField(n, t) for n, t in zip(names, tpe)]\n            )  # type: types.StructType\n\n    def __repr__(self):\n        return \"DataFrameType[{}]\".format(self.tpe)\n\n\n# The type is a scalar type that is furthermore understood by Spark.\nclass ScalarType(object):\n    def __init__(self, tpe):\n        self.tpe = tpe  # type: types.DataType\n\n    def __repr__(self):\n        return \"ScalarType[{}]\".format(self.tpe)\n\n\n# The type is left unspecified or we do not know about this type.\nclass UnknownType(object):\n    def __init__(self, tpe):\n        self.tpe = tpe\n\n    def __repr__(self):\n        return \"UnknownType[{}]\".format(self.tpe)\n\n\nclass NameTypeHolder(object):\n    name = None\n    tpe = None\n\n\ndef as_spark_type(tpe) -> types.DataType:\n    \"\"\"\n    Given a Python type, returns the equivalent spark type.\n    Accepts:\n    - the built-in types in Python\n    - the built-in types in numpy\n    - list of pairs of (field_name, type)\n    - dictionaries of field_name -> type\n    - Python3's typing system\n    \"\"\"\n    # TODO: Add \"boolean\" and \"string\" types.\n    # ArrayType\n    if tpe in (np.ndarray,):\n        return types.ArrayType(types.StringType())\n    elif hasattr(tpe, \"__origin__\") and issubclass(tpe.__origin__, list):\n        return types.ArrayType(as_spark_type(tpe.__args__[0]))\n    # BinaryType\n    elif tpe in (bytes, np.character, np.bytes_, np.string_):\n        return types.BinaryType()\n    # BooleanType\n    elif tpe in (bool, np.bool, \"bool\", \"?\"):\n        return types.BooleanType()\n    # DateType\n    elif tpe in (datetime.date,):\n        return types.DateType()\n    # NumericType\n    elif tpe in (np.int8, np.byte, \"int8\", \"byte\", \"b\"):\n        return types.ByteType()\n    elif tpe in (decimal.Decimal,):\n        # TODO: considering about the precision & scale for decimal type.\n        return types.DecimalType(38, 18)\n    elif tpe in (float, np.float, np.float64, \"float\", \"float64\", \"double\"):\n        return types.DoubleType()\n    elif tpe in (np.float32, \"float32\", \"f\"):\n        return types.FloatType()\n    elif tpe in (np.int32, \"int32\", \"i\"):\n        return types.IntegerType()\n    elif tpe in (int, np.int, np.int64, \"int\", \"int64\", \"long\", \"bigint\"):\n        return types.LongType()\n    elif tpe in (np.int16, \"int16\", \"short\"):\n        return types.ShortType()\n    # StringType\n    elif tpe in (str, np.unicode_, \"str\", \"U\"):\n        return types.StringType()\n    # TimestampType\n    elif tpe in (datetime.datetime, np.datetime64, \"datetime64[ns]\", \"M\"):\n        return types.TimestampType()\n    else:\n        raise TypeError(\"Type %s was not understood.\" % tpe)\n\n\ndef spark_type_to_pandas_dtype(spark_type):\n    \"\"\" Return the given Spark DataType to pandas dtype. \"\"\"\n    if isinstance(spark_type, (types.DateType, types.StructType, types.UserDefinedType)):\n        return np.dtype(\"object\")\n    elif isinstance(spark_type, types.TimestampType):\n        return np.dtype(\"datetime64[ns]\")\n    else:\n        return np.dtype(to_arrow_type(spark_type).to_pandas_dtype())\n\n\ndef infer_pd_series_spark_type(s: pd.Series) -> types.DataType:\n    \"\"\"Infer Spark DataType from pandas Series dtype.\n\n    :param s: :class:`pandas.Series` to be inferred\n    :return: the inferred Spark data type\n    \"\"\"\n    dt = s.dtype\n    if dt == np.dtype(\"object\"):\n        if len(s) == 0 or s.isnull().all():\n            raise ValueError(\"can not infer schema from empty or null dataset\")\n        elif hasattr(s[0], \"__UDT__\"):\n            return s[0].__UDT__\n        else:\n            return from_arrow_type(pa.Array.from_pandas(s).type)\n    elif is_datetime64_dtype(dt) or is_datetime64tz_dtype(dt):\n        return types.TimestampType()\n    else:\n        return from_arrow_type(pa.from_numpy_dtype(dt))\n\n\ndef infer_return_type(f) -> typing.Union[SeriesType, DataFrameType, ScalarType, UnknownType]:\n    \"\"\"\n    >>> def func() -> int:\n    ...    pass\n    >>> infer_return_type(func).tpe\n    LongType\n\n    >>> def func() -> ks.Series[int]:\n    ...    pass\n    >>> infer_return_type(func).tpe\n    LongType\n\n    >>> def func() -> ks.DataFrame[np.float, str]:\n    ...    pass\n    >>> infer_return_type(func).tpe\n    StructType(List(StructField(c0,DoubleType,true),StructField(c1,StringType,true)))\n\n    >>> def func() -> ks.DataFrame[np.float]:\n    ...    pass\n    >>> infer_return_type(func).tpe\n    StructType(List(StructField(c0,DoubleType,true)))\n\n    >>> def func() -> 'int':\n    ...    pass\n    >>> infer_return_type(func).tpe\n    LongType\n\n    >>> def func() -> 'ks.Series[int]':\n    ...    pass\n    >>> infer_return_type(func).tpe\n    LongType\n\n    >>> def func() -> 'ks.DataFrame[np.float, str]':\n    ...    pass\n    >>> infer_return_type(func).tpe\n    StructType(List(StructField(c0,DoubleType,true),StructField(c1,StringType,true)))\n\n    >>> def func() -> 'ks.DataFrame[np.float]':\n    ...    pass\n    >>> infer_return_type(func).tpe\n    StructType(List(StructField(c0,DoubleType,true)))\n\n    >>> def func() -> ks.DataFrame['a': np.float, 'b': int]:\n    ...     pass\n    >>> infer_return_type(func).tpe\n    StructType(List(StructField(a,DoubleType,true),StructField(b,LongType,true)))\n\n    >>> def func() -> \"ks.DataFrame['a': np.float, 'b': int]\":\n    ...     pass\n    >>> infer_return_type(func).tpe\n    StructType(List(StructField(a,DoubleType,true),StructField(b,LongType,true)))\n\n    >>> pdf = pd.DataFrame({\"a\": [1, 2, 3], \"b\": [3, 4, 5]})\n    >>> def func() -> ks.DataFrame[pdf.dtypes]:\n    ...     pass\n    >>> infer_return_type(func).tpe\n    StructType(List(StructField(c0,LongType,true),StructField(c1,LongType,true)))\n\n    >>> pdf = pd.DataFrame({\"a\": [1, 2, 3], \"b\": [3, 4, 5]})\n    >>> def func() -> ks.DataFrame[zip(pdf.columns, pdf.dtypes)]:\n    ...     pass\n    >>> infer_return_type(func).tpe\n    StructType(List(StructField(a,LongType,true),StructField(b,LongType,true)))\n    \"\"\"\n    # We should re-import to make sure the class 'SeriesType' is not treated as a class\n    # within this module locally. See Series.__class_getitem__ which imports this class\n    # canonically.\n    from databricks.koalas.typedef import SeriesType, NameTypeHolder\n\n    spec = getfullargspec(f)\n    tpe = spec.annotations.get(\"return\", None)\n    if isinstance(tpe, str):\n        # This type hint can happen when given hints are string to avoid forward reference.\n        tpe = resolve_string_type_hint(tpe)\n    if hasattr(tpe, \"__origin__\") and (\n        issubclass(tpe.__origin__, SeriesType) or tpe.__origin__ == ks.Series\n    ):\n        # TODO: remove \"tpe.__origin__ == ks.Series\" when we drop Python 3.5 and 3.6.\n        inner = as_spark_type(tpe.__args__[0])\n        return SeriesType(inner)\n\n    if hasattr(tpe, \"__origin__\") and tpe.__origin__ == ks.DataFrame:\n        # When Python version is lower then 3.7. Unwrap it to a Tuple type\n        # hints.\n        tpe = tpe.__args__[0]\n\n    # Note that, DataFrame type hints will create a Tuple.\n    # Python 3.6 has `__name__`. Python 3.7 and 3.8 have `_name`.\n    # Check if the name is Tuple.\n    name = getattr(tpe, \"_name\", getattr(tpe, \"__name__\", None))\n    if name == \"Tuple\":\n        tuple_type = tpe\n        if hasattr(tuple_type, \"__tuple_params__\"):\n            # Python 3.5.0 to 3.5.2 has '__tuple_params__' instead.\n            # See https://github.com/python/cpython/blob/v3.5.2/Lib/typing.py\n            parameters = getattr(tuple_type, \"__tuple_params__\")\n        else:\n            parameters = getattr(tuple_type, \"__args__\")\n        if len(parameters) > 0 and all(\n            isclass(p) and issubclass(p, NameTypeHolder) for p in parameters\n        ):\n            names = [p.name for p in parameters if issubclass(p, NameTypeHolder)]\n            types = [p.tpe for p in parameters if issubclass(p, NameTypeHolder)]\n            return DataFrameType([as_spark_type(t) for t in types], names)\n        return DataFrameType([as_spark_type(t) for t in parameters])\n    inner = as_spark_type(tpe)\n    if inner is None:\n        return UnknownType(tpe)\n    else:\n        return ScalarType(inner)\n", "path": "databricks/koalas/typedef/typehints.py"}]}