We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Tokenizer failed with unicode text
Tokenizer failed with unicode text 

d = [u'ali veli kırk dokuz elli',u'ali veli kırk dokuz elli veli kırk dokuz']
tokenizer.fit_on_texts(d)

TypeError                                 Traceback (most recent call last)
<ipython-input-181-b9cd26967244> in <module>()
----> 1 tokenizer.fit_on_texts(d)

C:\Anaconda2\lib\site-packages\keras-2.0.3-py2.7.egg\keras\preprocessing\text.pyc in fit_on_texts(self, texts)
    117                                                                      self.filters,
    118                                                                      self.lower,
--> 119                                                                      self.split)
    120             for w in seq:
    121                 if w in self.word_counts:

C:\Anaconda2\lib\site-packages\keras-2.0.3-py2.7.egg\keras\preprocessing\text.pyc in text_to_word_sequence(text, filters, lower, split)
     36     if lower:
     37         text = text.lower()
---> 38     text = text.translate(maketrans(filters, split * len(filters)))
     39     seq = text.split(split)
     40     return [i for i in seq if i]

TypeError: character mapping must return integer, None or unicode



--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `keras/preprocessing/text.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 """Utilities for text input preprocessing.
3 
4 May benefit from a fast Cython rewrite.
5 """
6 from __future__ import absolute_import
7 from __future__ import division
8 
9 import string
10 import sys
11 import warnings
12 from collections import OrderedDict
13 from hashlib import md5
14 
15 import numpy as np
16 from six.moves import range
17 from six.moves import zip
18 
19 if sys.version_info < (3,):
20     maketrans = string.maketrans
21 else:
22     maketrans = str.maketrans
23 
24 
25 def text_to_word_sequence(text,
26                           filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
27                           lower=True, split=" "):
28     """Converts a text to a sequence of words (or tokens).
29 
30     # Arguments
31         text: Input text (string).
32         filters: Sequence of characters to filter out.
33         lower: Whether to convert the input to lowercase.
34         split: Sentence split marker (string).
35 
36     # Returns
37         A list of words (or tokens).
38     """
39     if lower:
40         text = text.lower()
41     text = text.translate(maketrans(filters, split * len(filters)))
42     seq = text.split(split)
43     return [i for i in seq if i]
44 
45 
46 def one_hot(text, n,
47             filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
48             lower=True,
49             split=' '):
50     """One-hot encodes a text into a list of word indexes of size n.
51 
52     This is a wrapper to the `hashing_trick` function using `hash` as the
53     hashing function, unicity of word to index mapping non-guaranteed.
54     """
55     return hashing_trick(text, n,
56                          hash_function=hash,
57                          filters=filters,
58                          lower=lower,
59                          split=split)
60 
61 
62 def hashing_trick(text, n,
63                   hash_function=None,
64                   filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
65                   lower=True,
66                   split=' '):
67     """Converts a text to a sequence of indexes in a fixed-size hashing space.
68 
69     # Arguments
70         text: Input text (string).
71         n: Dimension of the hashing space.
72         hash_function: if `None` uses python `hash` function, can be 'md5' or
73             any function that takes in input a string and returns a int.
74             Note that `hash` is not a stable hashing function, so
75             it is not consistent across different runs, while 'md5'
76             is a stable hashing function.
77         filters: Sequence of characters to filter out.
78         lower: Whether to convert the input to lowercase.
79         split: Sentence split marker (string).
80 
81     # Returns
82         A list of integer word indices (unicity non-guaranteed).
83 
84     `0` is a reserved index that won't be assigned to any word.
85 
86     Two or more words may be assigned to the same index, due to possible
87     collisions by the hashing function.
88     The [probability](https://en.wikipedia.org/wiki/Birthday_problem#Probability_table)
89     of a collision is in relation to the dimension of the hashing space and
90     the number of distinct objects.
91     """
92     if hash_function is None:
93         hash_function = hash
94     elif hash_function == 'md5':
95         hash_function = lambda w: int(md5(w.encode()).hexdigest(), 16)
96 
97     seq = text_to_word_sequence(text,
98                                 filters=filters,
99                                 lower=lower,
100                                 split=split)
101     return [(hash_function(w) % (n - 1) + 1) for w in seq]
102 
103 
104 class Tokenizer(object):
105     """Text tokenization utility class.
106 
107     This class allows to vectorize a text corpus, by turning each
108     text into either a sequence of integers (each integer being the index
109     of a token in a dictionary) or into a vector where the coefficient
110     for each token could be binary, based on word count, based on tf-idf...
111 
112     # Arguments
113         num_words: the maximum number of words to keep, based
114             on word frequency. Only the most common `num_words` words will
115             be kept.
116         filters: a string where each element is a character that will be
117             filtered from the texts. The default is all punctuation, plus
118             tabs and line breaks, minus the `'` character.
119         lower: boolean. Whether to convert the texts to lowercase.
120         split: character or string to use for token splitting.
121         char_level: if True, every character will be treated as a token.
122 
123     By default, all punctuation is removed, turning the texts into
124     space-separated sequences of words
125     (words maybe include the `'` character). These sequences are then
126     split into lists of tokens. They will then be indexed or vectorized.
127 
128     `0` is a reserved index that won't be assigned to any word.
129     """
130 
131     def __init__(self, num_words=None,
132                  filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
133                  lower=True,
134                  split=' ',
135                  char_level=False,
136                  **kwargs):
137         # Legacy support
138         if 'nb_words' in kwargs:
139             warnings.warn('The `nb_words` argument in `Tokenizer` '
140                           'has been renamed `num_words`.')
141             num_words = kwargs.pop('nb_words')
142         if kwargs:
143             raise TypeError('Unrecognized keyword arguments: ' + str(kwargs))
144 
145         self.word_counts = OrderedDict()
146         self.word_docs = {}
147         self.filters = filters
148         self.split = split
149         self.lower = lower
150         self.num_words = num_words
151         self.document_count = 0
152         self.char_level = char_level
153 
154     def fit_on_texts(self, texts):
155         """Updates internal vocabulary based on a list of texts.
156 
157         Required before using `texts_to_sequences` or `texts_to_matrix`.
158 
159         # Arguments
160             texts: can be a list of strings,
161                 or a generator of strings (for memory-efficiency)
162         """
163         self.document_count = 0
164         for text in texts:
165             self.document_count += 1
166             seq = text if self.char_level else text_to_word_sequence(text,
167                                                                      self.filters,
168                                                                      self.lower,
169                                                                      self.split)
170             for w in seq:
171                 if w in self.word_counts:
172                     self.word_counts[w] += 1
173                 else:
174                     self.word_counts[w] = 1
175             for w in set(seq):
176                 if w in self.word_docs:
177                     self.word_docs[w] += 1
178                 else:
179                     self.word_docs[w] = 1
180 
181         wcounts = list(self.word_counts.items())
182         wcounts.sort(key=lambda x: x[1], reverse=True)
183         sorted_voc = [wc[0] for wc in wcounts]
184         # note that index 0 is reserved, never assigned to an existing word
185         self.word_index = dict(list(zip(sorted_voc, list(range(1, len(sorted_voc) + 1)))))
186 
187         self.index_docs = {}
188         for w, c in list(self.word_docs.items()):
189             self.index_docs[self.word_index[w]] = c
190 
191     def fit_on_sequences(self, sequences):
192         """Updates internal vocabulary based on a list of sequences.
193 
194         Required before using `sequences_to_matrix`
195         (if `fit_on_texts` was never called).
196 
197         # Arguments
198             sequences: A list of sequence.
199                 A "sequence" is a list of integer word indices.
200         """
201         self.document_count = len(sequences)
202         self.index_docs = {}
203         for seq in sequences:
204             seq = set(seq)
205             for i in seq:
206                 if i not in self.index_docs:
207                     self.index_docs[i] = 1
208                 else:
209                     self.index_docs[i] += 1
210 
211     def texts_to_sequences(self, texts):
212         """Transforms each text in texts in a sequence of integers.
213 
214         Only top "num_words" most frequent words will be taken into account.
215         Only words known by the tokenizer will be taken into account.
216 
217         # Arguments
218             texts: A list of texts (strings).
219 
220         # Returns
221             A list of sequences.
222         """
223         res = []
224         for vect in self.texts_to_sequences_generator(texts):
225             res.append(vect)
226         return res
227 
228     def texts_to_sequences_generator(self, texts):
229         """Transforms each text in texts in a sequence of integers.
230 
231         Only top "num_words" most frequent words will be taken into account.
232         Only words known by the tokenizer will be taken into account.
233 
234         # Arguments
235             texts: A list of texts (strings).
236 
237         # Yields
238             Yields individual sequences.
239         """
240         num_words = self.num_words
241         for text in texts:
242             seq = text if self.char_level else text_to_word_sequence(text,
243                                                                      self.filters,
244                                                                      self.lower,
245                                                                      self.split)
246             vect = []
247             for w in seq:
248                 i = self.word_index.get(w)
249                 if i is not None:
250                     if num_words and i >= num_words:
251                         continue
252                     else:
253                         vect.append(i)
254             yield vect
255 
256     def texts_to_matrix(self, texts, mode='binary'):
257         """Convert a list of texts to a Numpy matrix.
258 
259         # Arguments
260             texts: list of strings.
261             mode: one of "binary", "count", "tfidf", "freq".
262 
263         # Returns
264             A Numpy matrix.
265         """
266         sequences = self.texts_to_sequences(texts)
267         return self.sequences_to_matrix(sequences, mode=mode)
268 
269     def sequences_to_matrix(self, sequences, mode='binary'):
270         """Converts a list of sequences into a Numpy matrix.
271 
272         # Arguments
273             sequences: list of sequences
274                 (a sequence is a list of integer word indices).
275             mode: one of "binary", "count", "tfidf", "freq"
276 
277         # Returns
278             A Numpy matrix.
279 
280         # Raises
281             ValueError: In case of invalid `mode` argument,
282                 or if the Tokenizer requires to be fit to sample data.
283         """
284         if not self.num_words:
285             if self.word_index:
286                 num_words = len(self.word_index) + 1
287             else:
288                 raise ValueError('Specify a dimension (num_words argument), '
289                                  'or fit on some text data first.')
290         else:
291             num_words = self.num_words
292 
293         if mode == 'tfidf' and not self.document_count:
294             raise ValueError('Fit the Tokenizer on some data '
295                              'before using tfidf mode.')
296 
297         x = np.zeros((len(sequences), num_words))
298         for i, seq in enumerate(sequences):
299             if not seq:
300                 continue
301             counts = {}
302             for j in seq:
303                 if j >= num_words:
304                     continue
305                 if j not in counts:
306                     counts[j] = 1.
307                 else:
308                     counts[j] += 1
309             for j, c in list(counts.items()):
310                 if mode == 'count':
311                     x[i][j] = c
312                 elif mode == 'freq':
313                     x[i][j] = c / len(seq)
314                 elif mode == 'binary':
315                     x[i][j] = 1
316                 elif mode == 'tfidf':
317                     # Use weighting scheme 2 in
318                     # https://en.wikipedia.org/wiki/Tf%E2%80%93idf
319                     tf = 1 + np.log(c)
320                     idf = np.log(1 + self.document_count /
321                                  (1 + self.index_docs.get(j, 0)))
322                     x[i][j] = tf * idf
323                 else:
324                     raise ValueError('Unknown vectorization mode:', mode)
325         return x
326
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/keras/preprocessing/text.py b/keras/preprocessing/text.py
--- a/keras/preprocessing/text.py
+++ b/keras/preprocessing/text.py
@@ -38,7 +38,13 @@
     """
     if lower:
         text = text.lower()
-    text = text.translate(maketrans(filters, split * len(filters)))
+
+    if sys.version_info < (3,) and isinstance(text, unicode):
+        translate_map = dict((ord(c), unicode(split)) for c in filters)
+    else:
+        translate_map = maketrans(filters, split * len(filters))
+
+    text = text.translate(translate_map)
     seq = text.split(split)
     return [i for i in seq if i]

{"golden_diff": "diff --git a/keras/preprocessing/text.py b/keras/preprocessing/text.py\n--- a/keras/preprocessing/text.py\n+++ b/keras/preprocessing/text.py\n@@ -38,7 +38,13 @@\n     \"\"\"\n     if lower:\n         text = text.lower()\n-    text = text.translate(maketrans(filters, split * len(filters)))\n+\n+    if sys.version_info < (3,) and isinstance(text, unicode):\n+        translate_map = dict((ord(c), unicode(split)) for c in filters)\n+    else:\n+        translate_map = maketrans(filters, split * len(filters))\n+\n+    text = text.translate(translate_map)\n     seq = text.split(split)\n     return [i for i in seq if i]\n", "issue": "Tokenizer failed with unicode text\nTokenizer failed with unicode text \r\n\r\nd = [u'ali veli k\u0131rk dokuz elli',u'ali veli k\u0131rk dokuz elli veli k\u0131rk dokuz']\r\ntokenizer.fit_on_texts(d)\r\n\r\nTypeError                                 Traceback (most recent call last)\r\n<ipython-input-181-b9cd26967244> in <module>()\r\n----> 1 tokenizer.fit_on_texts(d)\r\n\r\nC:\\Anaconda2\\lib\\site-packages\\keras-2.0.3-py2.7.egg\\keras\\preprocessing\\text.pyc in fit_on_texts(self, texts)\r\n    117                                                                      self.filters,\r\n    118                                                                      self.lower,\r\n--> 119                                                                      self.split)\r\n    120             for w in seq:\r\n    121                 if w in self.word_counts:\r\n\r\nC:\\Anaconda2\\lib\\site-packages\\keras-2.0.3-py2.7.egg\\keras\\preprocessing\\text.pyc in text_to_word_sequence(text, filters, lower, split)\r\n     36     if lower:\r\n     37         text = text.lower()\r\n---> 38     text = text.translate(maketrans(filters, split * len(filters)))\r\n     39     seq = text.split(split)\r\n     40     return [i for i in seq if i]\r\n\r\nTypeError: character mapping must return integer, None or unicode\r\n\r\n\n", "before_files": [{"content": "# -*- coding: utf-8 -*-\n\"\"\"Utilities for text input preprocessing.\n\nMay benefit from a fast Cython rewrite.\n\"\"\"\nfrom __future__ import absolute_import\nfrom __future__ import division\n\nimport string\nimport sys\nimport warnings\nfrom collections import OrderedDict\nfrom hashlib import md5\n\nimport numpy as np\nfrom six.moves import range\nfrom six.moves import zip\n\nif sys.version_info < (3,):\n    maketrans = string.maketrans\nelse:\n    maketrans = str.maketrans\n\n\ndef text_to_word_sequence(text,\n                          filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',\n                          lower=True, split=\" \"):\n    \"\"\"Converts a text to a sequence of words (or tokens).\n\n    # Arguments\n        text: Input text (string).\n        filters: Sequence of characters to filter out.\n        lower: Whether to convert the input to lowercase.\n        split: Sentence split marker (string).\n\n    # Returns\n        A list of words (or tokens).\n    \"\"\"\n    if lower:\n        text = text.lower()\n    text = text.translate(maketrans(filters, split * len(filters)))\n    seq = text.split(split)\n    return [i for i in seq if i]\n\n\ndef one_hot(text, n,\n            filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',\n            lower=True,\n            split=' '):\n    \"\"\"One-hot encodes a text into a list of word indexes of size n.\n\n    This is a wrapper to the `hashing_trick` function using `hash` as the\n    hashing function, unicity of word to index mapping non-guaranteed.\n    \"\"\"\n    return hashing_trick(text, n,\n                         hash_function=hash,\n                         filters=filters,\n                         lower=lower,\n                         split=split)\n\n\ndef hashing_trick(text, n,\n                  hash_function=None,\n                  filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',\n                  lower=True,\n                  split=' '):\n    \"\"\"Converts a text to a sequence of indexes in a fixed-size hashing space.\n\n    # Arguments\n        text: Input text (string).\n        n: Dimension of the hashing space.\n        hash_function: if `None` uses python `hash` function, can be 'md5' or\n            any function that takes in input a string and returns a int.\n            Note that `hash` is not a stable hashing function, so\n            it is not consistent across different runs, while 'md5'\n            is a stable hashing function.\n        filters: Sequence of characters to filter out.\n        lower: Whether to convert the input to lowercase.\n        split: Sentence split marker (string).\n\n    # Returns\n        A list of integer word indices (unicity non-guaranteed).\n\n    `0` is a reserved index that won't be assigned to any word.\n\n    Two or more words may be assigned to the same index, due to possible\n    collisions by the hashing function.\n    The [probability](https://en.wikipedia.org/wiki/Birthday_problem#Probability_table)\n    of a collision is in relation to the dimension of the hashing space and\n    the number of distinct objects.\n    \"\"\"\n    if hash_function is None:\n        hash_function = hash\n    elif hash_function == 'md5':\n        hash_function = lambda w: int(md5(w.encode()).hexdigest(), 16)\n\n    seq = text_to_word_sequence(text,\n                                filters=filters,\n                                lower=lower,\n                                split=split)\n    return [(hash_function(w) % (n - 1) + 1) for w in seq]\n\n\nclass Tokenizer(object):\n    \"\"\"Text tokenization utility class.\n\n    This class allows to vectorize a text corpus, by turning each\n    text into either a sequence of integers (each integer being the index\n    of a token in a dictionary) or into a vector where the coefficient\n    for each token could be binary, based on word count, based on tf-idf...\n\n    # Arguments\n        num_words: the maximum number of words to keep, based\n            on word frequency. Only the most common `num_words` words will\n            be kept.\n        filters: a string where each element is a character that will be\n            filtered from the texts. The default is all punctuation, plus\n            tabs and line breaks, minus the `'` character.\n        lower: boolean. Whether to convert the texts to lowercase.\n        split: character or string to use for token splitting.\n        char_level: if True, every character will be treated as a token.\n\n    By default, all punctuation is removed, turning the texts into\n    space-separated sequences of words\n    (words maybe include the `'` character). These sequences are then\n    split into lists of tokens. They will then be indexed or vectorized.\n\n    `0` is a reserved index that won't be assigned to any word.\n    \"\"\"\n\n    def __init__(self, num_words=None,\n                 filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',\n                 lower=True,\n                 split=' ',\n                 char_level=False,\n                 **kwargs):\n        # Legacy support\n        if 'nb_words' in kwargs:\n            warnings.warn('The `nb_words` argument in `Tokenizer` '\n                          'has been renamed `num_words`.')\n            num_words = kwargs.pop('nb_words')\n        if kwargs:\n            raise TypeError('Unrecognized keyword arguments: ' + str(kwargs))\n\n        self.word_counts = OrderedDict()\n        self.word_docs = {}\n        self.filters = filters\n        self.split = split\n        self.lower = lower\n        self.num_words = num_words\n        self.document_count = 0\n        self.char_level = char_level\n\n    def fit_on_texts(self, texts):\n        \"\"\"Updates internal vocabulary based on a list of texts.\n\n        Required before using `texts_to_sequences` or `texts_to_matrix`.\n\n        # Arguments\n            texts: can be a list of strings,\n                or a generator of strings (for memory-efficiency)\n        \"\"\"\n        self.document_count = 0\n        for text in texts:\n            self.document_count += 1\n            seq = text if self.char_level else text_to_word_sequence(text,\n                                                                     self.filters,\n                                                                     self.lower,\n                                                                     self.split)\n            for w in seq:\n                if w in self.word_counts:\n                    self.word_counts[w] += 1\n                else:\n                    self.word_counts[w] = 1\n            for w in set(seq):\n                if w in self.word_docs:\n                    self.word_docs[w] += 1\n                else:\n                    self.word_docs[w] = 1\n\n        wcounts = list(self.word_counts.items())\n        wcounts.sort(key=lambda x: x[1], reverse=True)\n        sorted_voc = [wc[0] for wc in wcounts]\n        # note that index 0 is reserved, never assigned to an existing word\n        self.word_index = dict(list(zip(sorted_voc, list(range(1, len(sorted_voc) + 1)))))\n\n        self.index_docs = {}\n        for w, c in list(self.word_docs.items()):\n            self.index_docs[self.word_index[w]] = c\n\n    def fit_on_sequences(self, sequences):\n        \"\"\"Updates internal vocabulary based on a list of sequences.\n\n        Required before using `sequences_to_matrix`\n        (if `fit_on_texts` was never called).\n\n        # Arguments\n            sequences: A list of sequence.\n                A \"sequence\" is a list of integer word indices.\n        \"\"\"\n        self.document_count = len(sequences)\n        self.index_docs = {}\n        for seq in sequences:\n            seq = set(seq)\n            for i in seq:\n                if i not in self.index_docs:\n                    self.index_docs[i] = 1\n                else:\n                    self.index_docs[i] += 1\n\n    def texts_to_sequences(self, texts):\n        \"\"\"Transforms each text in texts in a sequence of integers.\n\n        Only top \"num_words\" most frequent words will be taken into account.\n        Only words known by the tokenizer will be taken into account.\n\n        # Arguments\n            texts: A list of texts (strings).\n\n        # Returns\n            A list of sequences.\n        \"\"\"\n        res = []\n        for vect in self.texts_to_sequences_generator(texts):\n            res.append(vect)\n        return res\n\n    def texts_to_sequences_generator(self, texts):\n        \"\"\"Transforms each text in texts in a sequence of integers.\n\n        Only top \"num_words\" most frequent words will be taken into account.\n        Only words known by the tokenizer will be taken into account.\n\n        # Arguments\n            texts: A list of texts (strings).\n\n        # Yields\n            Yields individual sequences.\n        \"\"\"\n        num_words = self.num_words\n        for text in texts:\n            seq = text if self.char_level else text_to_word_sequence(text,\n                                                                     self.filters,\n                                                                     self.lower,\n                                                                     self.split)\n            vect = []\n            for w in seq:\n                i = self.word_index.get(w)\n                if i is not None:\n                    if num_words and i >= num_words:\n                        continue\n                    else:\n                        vect.append(i)\n            yield vect\n\n    def texts_to_matrix(self, texts, mode='binary'):\n        \"\"\"Convert a list of texts to a Numpy matrix.\n\n        # Arguments\n            texts: list of strings.\n            mode: one of \"binary\", \"count\", \"tfidf\", \"freq\".\n\n        # Returns\n            A Numpy matrix.\n        \"\"\"\n        sequences = self.texts_to_sequences(texts)\n        return self.sequences_to_matrix(sequences, mode=mode)\n\n    def sequences_to_matrix(self, sequences, mode='binary'):\n        \"\"\"Converts a list of sequences into a Numpy matrix.\n\n        # Arguments\n            sequences: list of sequences\n                (a sequence is a list of integer word indices).\n            mode: one of \"binary\", \"count\", \"tfidf\", \"freq\"\n\n        # Returns\n            A Numpy matrix.\n\n        # Raises\n            ValueError: In case of invalid `mode` argument,\n                or if the Tokenizer requires to be fit to sample data.\n        \"\"\"\n        if not self.num_words:\n            if self.word_index:\n                num_words = len(self.word_index) + 1\n            else:\n                raise ValueError('Specify a dimension (num_words argument), '\n                                 'or fit on some text data first.')\n        else:\n            num_words = self.num_words\n\n        if mode == 'tfidf' and not self.document_count:\n            raise ValueError('Fit the Tokenizer on some data '\n                             'before using tfidf mode.')\n\n        x = np.zeros((len(sequences), num_words))\n        for i, seq in enumerate(sequences):\n            if not seq:\n                continue\n            counts = {}\n            for j in seq:\n                if j >= num_words:\n                    continue\n                if j not in counts:\n                    counts[j] = 1.\n                else:\n                    counts[j] += 1\n            for j, c in list(counts.items()):\n                if mode == 'count':\n                    x[i][j] = c\n                elif mode == 'freq':\n                    x[i][j] = c / len(seq)\n                elif mode == 'binary':\n                    x[i][j] = 1\n                elif mode == 'tfidf':\n                    # Use weighting scheme 2 in\n                    # https://en.wikipedia.org/wiki/Tf%E2%80%93idf\n                    tf = 1 + np.log(c)\n                    idf = np.log(1 + self.document_count /\n                                 (1 + self.index_docs.get(j, 0)))\n                    x[i][j] = tf * idf\n                else:\n                    raise ValueError('Unknown vectorization mode:', mode)\n        return x\n", "path": "keras/preprocessing/text.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\n\"\"\"Utilities for text input preprocessing.\n\nMay benefit from a fast Cython rewrite.\n\"\"\"\nfrom __future__ import absolute_import\nfrom __future__ import division\n\nimport string\nimport sys\nimport warnings\nfrom collections import OrderedDict\nfrom hashlib import md5\n\nimport numpy as np\nfrom six.moves import range\nfrom six.moves import zip\n\nif sys.version_info < (3,):\n    maketrans = string.maketrans\nelse:\n    maketrans = str.maketrans\n\n\ndef text_to_word_sequence(text,\n                          filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',\n                          lower=True, split=\" \"):\n    \"\"\"Converts a text to a sequence of words (or tokens).\n\n    # Arguments\n        text: Input text (string).\n        filters: Sequence of characters to filter out.\n        lower: Whether to convert the input to lowercase.\n        split: Sentence split marker (string).\n\n    # Returns\n        A list of words (or tokens).\n    \"\"\"\n    if lower:\n        text = text.lower()\n\n    if sys.version_info < (3,) and isinstance(text, unicode):\n        translate_map = dict((ord(c), unicode(split)) for c in filters)\n    else:\n        translate_map = maketrans(filters, split * len(filters))\n\n    text = text.translate(translate_map)\n    seq = text.split(split)\n    return [i for i in seq if i]\n\n\ndef one_hot(text, n,\n            filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',\n            lower=True,\n            split=' '):\n    \"\"\"One-hot encodes a text into a list of word indexes of size n.\n\n    This is a wrapper to the `hashing_trick` function using `hash` as the\n    hashing function, unicity of word to index mapping non-guaranteed.\n    \"\"\"\n    return hashing_trick(text, n,\n                         hash_function=hash,\n                         filters=filters,\n                         lower=lower,\n                         split=split)\n\n\ndef hashing_trick(text, n,\n                  hash_function=None,\n                  filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',\n                  lower=True,\n                  split=' '):\n    \"\"\"Converts a text to a sequence of indexes in a fixed-size hashing space.\n\n    # Arguments\n        text: Input text (string).\n        n: Dimension of the hashing space.\n        hash_function: if `None` uses python `hash` function, can be 'md5' or\n            any function that takes in input a string and returns a int.\n            Note that `hash` is not a stable hashing function, so\n            it is not consistent across different runs, while 'md5'\n            is a stable hashing function.\n        filters: Sequence of characters to filter out.\n        lower: Whether to convert the input to lowercase.\n        split: Sentence split marker (string).\n\n    # Returns\n        A list of integer word indices (unicity non-guaranteed).\n\n    `0` is a reserved index that won't be assigned to any word.\n\n    Two or more words may be assigned to the same index, due to possible\n    collisions by the hashing function.\n    The [probability](https://en.wikipedia.org/wiki/Birthday_problem#Probability_table)\n    of a collision is in relation to the dimension of the hashing space and\n    the number of distinct objects.\n    \"\"\"\n    if hash_function is None:\n        hash_function = hash\n    elif hash_function == 'md5':\n        hash_function = lambda w: int(md5(w.encode()).hexdigest(), 16)\n\n    seq = text_to_word_sequence(text,\n                                filters=filters,\n                                lower=lower,\n                                split=split)\n    return [(hash_function(w) % (n - 1) + 1) for w in seq]\n\n\nclass Tokenizer(object):\n    \"\"\"Text tokenization utility class.\n\n    This class allows to vectorize a text corpus, by turning each\n    text into either a sequence of integers (each integer being the index\n    of a token in a dictionary) or into a vector where the coefficient\n    for each token could be binary, based on word count, based on tf-idf...\n\n    # Arguments\n        num_words: the maximum number of words to keep, based\n            on word frequency. Only the most common `num_words` words will\n            be kept.\n        filters: a string where each element is a character that will be\n            filtered from the texts. The default is all punctuation, plus\n            tabs and line breaks, minus the `'` character.\n        lower: boolean. Whether to convert the texts to lowercase.\n        split: character or string to use for token splitting.\n        char_level: if True, every character will be treated as a token.\n\n    By default, all punctuation is removed, turning the texts into\n    space-separated sequences of words\n    (words maybe include the `'` character). These sequences are then\n    split into lists of tokens. They will then be indexed or vectorized.\n\n    `0` is a reserved index that won't be assigned to any word.\n    \"\"\"\n\n    def __init__(self, num_words=None,\n                 filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',\n                 lower=True,\n                 split=' ',\n                 char_level=False,\n                 **kwargs):\n        # Legacy support\n        if 'nb_words' in kwargs:\n            warnings.warn('The `nb_words` argument in `Tokenizer` '\n                          'has been renamed `num_words`.')\n            num_words = kwargs.pop('nb_words')\n        if kwargs:\n            raise TypeError('Unrecognized keyword arguments: ' + str(kwargs))\n\n        self.word_counts = OrderedDict()\n        self.word_docs = {}\n        self.filters = filters\n        self.split = split\n        self.lower = lower\n        self.num_words = num_words\n        self.document_count = 0\n        self.char_level = char_level\n\n    def fit_on_texts(self, texts):\n        \"\"\"Updates internal vocabulary based on a list of texts.\n\n        Required before using `texts_to_sequences` or `texts_to_matrix`.\n\n        # Arguments\n            texts: can be a list of strings,\n                or a generator of strings (for memory-efficiency)\n        \"\"\"\n        self.document_count = 0\n        for text in texts:\n            self.document_count += 1\n            seq = text if self.char_level else text_to_word_sequence(text,\n                                                                     self.filters,\n                                                                     self.lower,\n                                                                     self.split)\n            for w in seq:\n                if w in self.word_counts:\n                    self.word_counts[w] += 1\n                else:\n                    self.word_counts[w] = 1\n            for w in set(seq):\n                if w in self.word_docs:\n                    self.word_docs[w] += 1\n                else:\n                    self.word_docs[w] = 1\n\n        wcounts = list(self.word_counts.items())\n        wcounts.sort(key=lambda x: x[1], reverse=True)\n        sorted_voc = [wc[0] for wc in wcounts]\n        # note that index 0 is reserved, never assigned to an existing word\n        self.word_index = dict(list(zip(sorted_voc, list(range(1, len(sorted_voc) + 1)))))\n\n        self.index_docs = {}\n        for w, c in list(self.word_docs.items()):\n            self.index_docs[self.word_index[w]] = c\n\n    def fit_on_sequences(self, sequences):\n        \"\"\"Updates internal vocabulary based on a list of sequences.\n\n        Required before using `sequences_to_matrix`\n        (if `fit_on_texts` was never called).\n\n        # Arguments\n            sequences: A list of sequence.\n                A \"sequence\" is a list of integer word indices.\n        \"\"\"\n        self.document_count = len(sequences)\n        self.index_docs = {}\n        for seq in sequences:\n            seq = set(seq)\n            for i in seq:\n                if i not in self.index_docs:\n                    self.index_docs[i] = 1\n                else:\n                    self.index_docs[i] += 1\n\n    def texts_to_sequences(self, texts):\n        \"\"\"Transforms each text in texts in a sequence of integers.\n\n        Only top \"num_words\" most frequent words will be taken into account.\n        Only words known by the tokenizer will be taken into account.\n\n        # Arguments\n            texts: A list of texts (strings).\n\n        # Returns\n            A list of sequences.\n        \"\"\"\n        res = []\n        for vect in self.texts_to_sequences_generator(texts):\n            res.append(vect)\n        return res\n\n    def texts_to_sequences_generator(self, texts):\n        \"\"\"Transforms each text in texts in a sequence of integers.\n\n        Only top \"num_words\" most frequent words will be taken into account.\n        Only words known by the tokenizer will be taken into account.\n\n        # Arguments\n            texts: A list of texts (strings).\n\n        # Yields\n            Yields individual sequences.\n        \"\"\"\n        num_words = self.num_words\n        for text in texts:\n            seq = text if self.char_level else text_to_word_sequence(text,\n                                                                     self.filters,\n                                                                     self.lower,\n                                                                     self.split)\n            vect = []\n            for w in seq:\n                i = self.word_index.get(w)\n                if i is not None:\n                    if num_words and i >= num_words:\n                        continue\n                    else:\n                        vect.append(i)\n            yield vect\n\n    def texts_to_matrix(self, texts, mode='binary'):\n        \"\"\"Convert a list of texts to a Numpy matrix.\n\n        # Arguments\n            texts: list of strings.\n            mode: one of \"binary\", \"count\", \"tfidf\", \"freq\".\n\n        # Returns\n            A Numpy matrix.\n        \"\"\"\n        sequences = self.texts_to_sequences(texts)\n        return self.sequences_to_matrix(sequences, mode=mode)\n\n    def sequences_to_matrix(self, sequences, mode='binary'):\n        \"\"\"Converts a list of sequences into a Numpy matrix.\n\n        # Arguments\n            sequences: list of sequences\n                (a sequence is a list of integer word indices).\n            mode: one of \"binary\", \"count\", \"tfidf\", \"freq\"\n\n        # Returns\n            A Numpy matrix.\n\n        # Raises\n            ValueError: In case of invalid `mode` argument,\n                or if the Tokenizer requires to be fit to sample data.\n        \"\"\"\n        if not self.num_words:\n            if self.word_index:\n                num_words = len(self.word_index) + 1\n            else:\n                raise ValueError('Specify a dimension (num_words argument), '\n                                 'or fit on some text data first.')\n        else:\n            num_words = self.num_words\n\n        if mode == 'tfidf' and not self.document_count:\n            raise ValueError('Fit the Tokenizer on some data '\n                             'before using tfidf mode.')\n\n        x = np.zeros((len(sequences), num_words))\n        for i, seq in enumerate(sequences):\n            if not seq:\n                continue\n            counts = {}\n            for j in seq:\n                if j >= num_words:\n                    continue\n                if j not in counts:\n                    counts[j] = 1.\n                else:\n                    counts[j] += 1\n            for j, c in list(counts.items()):\n                if mode == 'count':\n                    x[i][j] = c\n                elif mode == 'freq':\n                    x[i][j] = c / len(seq)\n                elif mode == 'binary':\n                    x[i][j] = 1\n                elif mode == 'tfidf':\n                    # Use weighting scheme 2 in\n                    # https://en.wikipedia.org/wiki/Tf%E2%80%93idf\n                    tf = 1 + np.log(c)\n                    idf = np.log(1 + self.document_count /\n                                 (1 + self.index_docs.get(j, 0)))\n                    x[i][j] = tf * idf\n                else:\n                    raise ValueError('Unknown vectorization mode:', mode)\n        return x\n", "path": "keras/preprocessing/text.py"}]}