We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Implementing __contains__ for vocab.Vectors class
## 🚀 Feature

Isn't it better to implement \_\_contains\_\_ for Vectors class? In this way, one can easily find out whether a vocab is in the self.itos or not. 

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `torchtext/vocab/vectors.py`
Content:
```
1 import gzip
2 import logging
3 import os
4 import tarfile
5 import zipfile
6 from functools import partial
7 from urllib.request import urlretrieve
8 
9 import torch
10 from tqdm import tqdm
11 
12 from ..utils import reporthook
13 
14 logger = logging.getLogger(__name__)
15 
16 
17 def _infer_shape(f):
18     num_lines, vector_dim = 0, None
19     for line in f:
20         if vector_dim is None:
21             row = line.rstrip().split(b" ")
22             vector = row[1:]
23             # Assuming word, [vector] format
24             if len(vector) > 2:
25                 # The header present in some (w2v) formats contains two elements.
26                 vector_dim = len(vector)
27                 num_lines += 1  # First element read
28         else:
29             num_lines += 1
30     f.seek(0)
31     return num_lines, vector_dim
32 
33 
34 class Vectors(object):
35     def __init__(self, name, cache=None, url=None, unk_init=None, max_vectors=None) -> None:
36         """
37         Args:
38 
39             name: name of the file that contains the vectors
40             cache: directory for cached vectors
41             url: url for download if vectors not found in cache
42             unk_init (callback): by default, initialize out-of-vocabulary word vectors
43                 to zero vectors; can be any function that takes in a Tensor and returns a Tensor of the same size
44             max_vectors (int): this can be used to limit the number of
45                 pre-trained vectors loaded.
46                 Most pre-trained vector sets are sorted
47                 in the descending order of word frequency.
48                 Thus, in situations where the entire set doesn't fit in memory,
49                 or is not needed for another reason, passing `max_vectors`
50                 can limit the size of the loaded set.
51         """
52 
53         cache = ".vector_cache" if cache is None else cache
54         self.itos = None
55         self.stoi = None
56         self.vectors = None
57         self.dim = None
58         self.unk_init = torch.Tensor.zero_ if unk_init is None else unk_init
59         self.cache(name, cache, url=url, max_vectors=max_vectors)
60 
61     def __getitem__(self, token):
62         if token in self.stoi:
63             return self.vectors[self.stoi[token]]
64         else:
65             return self.unk_init(torch.Tensor(self.dim))
66 
67     def cache(self, name, cache, url=None, max_vectors=None):
68         import ssl
69 
70         ssl._create_default_https_context = ssl._create_unverified_context
71         if os.path.isfile(name):
72             path = name
73             if max_vectors:
74                 file_suffix = "_{}.pt".format(max_vectors)
75             else:
76                 file_suffix = ".pt"
77             path_pt = os.path.join(cache, os.path.basename(name)) + file_suffix
78         else:
79             path = os.path.join(cache, name)
80             if max_vectors:
81                 file_suffix = "_{}.pt".format(max_vectors)
82             else:
83                 file_suffix = ".pt"
84             path_pt = path + file_suffix
85 
86         if not os.path.isfile(path_pt):
87             if not os.path.isfile(path) and url:
88                 logger.info("Downloading vectors from {}".format(url))
89                 if not os.path.exists(cache):
90                     os.makedirs(cache)
91                 dest = os.path.join(cache, os.path.basename(url))
92                 if not os.path.isfile(dest):
93                     with tqdm(unit="B", unit_scale=True, miniters=1, desc=dest) as t:
94                         try:
95                             urlretrieve(url, dest, reporthook=reporthook(t))
96                         except KeyboardInterrupt as e:  # remove the partial zip file
97                             os.remove(dest)
98                             raise e
99                 logger.info("Extracting vectors into {}".format(cache))
100                 ext = os.path.splitext(dest)[1][1:]
101                 if ext == "zip":
102                     with zipfile.ZipFile(dest, "r") as zf:
103                         zf.extractall(cache)
104                 elif ext == "gz":
105                     if dest.endswith(".tar.gz"):
106                         with tarfile.open(dest, "r:gz") as tar:
107                             tar.extractall(path=cache)
108             if not os.path.isfile(path):
109                 raise RuntimeError("no vectors found at {}".format(path))
110 
111             logger.info("Loading vectors from {}".format(path))
112             ext = os.path.splitext(path)[1][1:]
113             if ext == "gz":
114                 open_file = gzip.open
115             else:
116                 open_file = open
117 
118             vectors_loaded = 0
119             with open_file(path, "rb") as f:
120                 num_lines, dim = _infer_shape(f)
121                 if not max_vectors or max_vectors > num_lines:
122                     max_vectors = num_lines
123 
124                 itos, vectors, dim = [], torch.zeros((max_vectors, dim)), None
125 
126                 for line in tqdm(f, total=max_vectors):
127                     # Explicitly splitting on " " is important, so we don't
128                     # get rid of Unicode non-breaking spaces in the vectors.
129                     entries = line.rstrip().split(b" ")
130 
131                     word, entries = entries[0], entries[1:]
132                     if dim is None and len(entries) > 1:
133                         dim = len(entries)
134                     elif len(entries) == 1:
135                         logger.warning(
136                             "Skipping token {} with 1-dimensional " "vector {}; likely a header".format(word, entries)
137                         )
138                         continue
139                     elif dim != len(entries):
140                         raise RuntimeError(
141                             "Vector for token {} has {} dimensions, but previously "
142                             "read vectors have {} dimensions. All vectors must have "
143                             "the same number of dimensions.".format(word, len(entries), dim)
144                         )
145 
146                     try:
147                         if isinstance(word, bytes):
148                             word = word.decode("utf-8")
149                     except UnicodeDecodeError:
150                         logger.info("Skipping non-UTF8 token {}".format(repr(word)))
151                         continue
152 
153                     vectors[vectors_loaded] = torch.tensor([float(x) for x in entries])
154                     vectors_loaded += 1
155                     itos.append(word)
156 
157                     if vectors_loaded == max_vectors:
158                         break
159 
160             self.itos = itos
161             self.stoi = {word: i for i, word in enumerate(itos)}
162             self.vectors = torch.Tensor(vectors).view(-1, dim)
163             self.dim = dim
164             logger.info("Saving vectors to {}".format(path_pt))
165             if not os.path.exists(cache):
166                 os.makedirs(cache)
167             torch.save((self.itos, self.stoi, self.vectors, self.dim), path_pt)
168         else:
169             logger.info("Loading vectors from {}".format(path_pt))
170             self.itos, self.stoi, self.vectors, self.dim = torch.load(path_pt)
171 
172     def __len__(self):
173         return len(self.vectors)
174 
175     def get_vecs_by_tokens(self, tokens, lower_case_backup=False):
176         """Look up embedding vectors of tokens.
177 
178         Args:
179             tokens: a token or a list of tokens. if `tokens` is a string,
180                 returns a 1-D tensor of shape `self.dim`; if `tokens` is a
181                 list of strings, returns a 2-D tensor of shape=(len(tokens),
182                 self.dim).
183             lower_case_backup : Whether to look up the token in the lower case.
184                 If False, each token in the original case will be looked up;
185                 if True, each token in the original case will be looked up first,
186                 if not found in the keys of the property `stoi`, the token in the
187                 lower case will be looked up. Default: False.
188 
189         Examples:
190             >>> examples = ['chip', 'baby', 'Beautiful']
191             >>> vec = text.vocab.GloVe(name='6B', dim=50)
192             >>> ret = vec.get_vecs_by_tokens(examples, lower_case_backup=True)
193         """
194         to_reduce = False
195 
196         if not isinstance(tokens, list):
197             tokens = [tokens]
198             to_reduce = True
199 
200         if not lower_case_backup:
201             indices = [self[token] for token in tokens]
202         else:
203             indices = [self[token] if token in self.stoi else self[token.lower()] for token in tokens]
204 
205         vecs = torch.stack(indices)
206         return vecs[0] if to_reduce else vecs
207 
208 
209 class GloVe(Vectors):
210     url = {
211         "42B": "http://nlp.stanford.edu/data/glove.42B.300d.zip",
212         "840B": "http://nlp.stanford.edu/data/glove.840B.300d.zip",
213         "twitter.27B": "http://nlp.stanford.edu/data/glove.twitter.27B.zip",
214         "6B": "http://nlp.stanford.edu/data/glove.6B.zip",
215     }
216 
217     def __init__(self, name="840B", dim=300, **kwargs) -> None:
218         url = self.url[name]
219         name = "glove.{}.{}d.txt".format(name, str(dim))
220         super(GloVe, self).__init__(name, url=url, **kwargs)
221 
222 
223 class FastText(Vectors):
224 
225     url_base = "https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.{}.vec"
226 
227     def __init__(self, language="en", **kwargs) -> None:
228         url = self.url_base.format(language)
229         name = os.path.basename(url)
230         super(FastText, self).__init__(name, url=url, **kwargs)
231 
232 
233 class CharNGram(Vectors):
234 
235     name = "charNgram.txt"
236     url = "http://www.logos.t.u-tokyo.ac.jp/~hassy/publications/arxiv2016jmt/" "jmt_pre-trained_embeddings.tar.gz"
237 
238     def __init__(self, **kwargs) -> None:
239         super(CharNGram, self).__init__(self.name, url=self.url, **kwargs)
240 
241     def __getitem__(self, token):
242         vector = torch.Tensor(1, self.dim).zero_()
243         if token == "<unk>":
244             return self.unk_init(vector)
245         chars = ["#BEGIN#"] + list(token) + ["#END#"]
246         num_vectors = 0
247         for n in [2, 3, 4]:
248             end = len(chars) - n + 1
249             grams = [chars[i : (i + n)] for i in range(end)]
250             for gram in grams:
251                 gram_key = "{}gram-{}".format(n, "".join(gram))
252                 if gram_key in self.stoi:
253                     vector += self.vectors[self.stoi[gram_key]]
254                     num_vectors += 1
255         if num_vectors > 0:
256             vector /= num_vectors
257         else:
258             vector = self.unk_init(vector)
259         return vector
260 
261 
262 pretrained_aliases = {
263     "charngram.100d": partial(CharNGram),
264     "fasttext.en.300d": partial(FastText, language="en"),
265     "fasttext.simple.300d": partial(FastText, language="simple"),
266     "glove.42B.300d": partial(GloVe, name="42B", dim="300"),
267     "glove.840B.300d": partial(GloVe, name="840B", dim="300"),
268     "glove.twitter.27B.25d": partial(GloVe, name="twitter.27B", dim="25"),
269     "glove.twitter.27B.50d": partial(GloVe, name="twitter.27B", dim="50"),
270     "glove.twitter.27B.100d": partial(GloVe, name="twitter.27B", dim="100"),
271     "glove.twitter.27B.200d": partial(GloVe, name="twitter.27B", dim="200"),
272     "glove.6B.50d": partial(GloVe, name="6B", dim="50"),
273     "glove.6B.100d": partial(GloVe, name="6B", dim="100"),
274     "glove.6B.200d": partial(GloVe, name="6B", dim="200"),
275     "glove.6B.300d": partial(GloVe, name="6B", dim="300"),
276 }
277 """Mapping from string name to factory function"""
278
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/torchtext/vocab/vectors.py b/torchtext/vocab/vectors.py
--- a/torchtext/vocab/vectors.py
+++ b/torchtext/vocab/vectors.py
@@ -64,6 +64,9 @@
         else:
             return self.unk_init(torch.Tensor(self.dim))
 
+    def __contains__(self, token):
+        return token in self.stoi
+
     def cache(self, name, cache, url=None, max_vectors=None):
         import ssl

{"golden_diff": "diff --git a/torchtext/vocab/vectors.py b/torchtext/vocab/vectors.py\n--- a/torchtext/vocab/vectors.py\n+++ b/torchtext/vocab/vectors.py\n@@ -64,6 +64,9 @@\n         else:\n             return self.unk_init(torch.Tensor(self.dim))\n \n+    def __contains__(self, token):\n+        return token in self.stoi\n+\n     def cache(self, name, cache, url=None, max_vectors=None):\n         import ssl\n", "issue": "Implementing __contains__ for vocab.Vectors class\n## \ud83d\ude80 Feature\r\n\r\nIsn't it better to implement \\_\\_contains\\_\\_ for Vectors class? In this way, one can easily find out whether a vocab is in the self.itos or not. \n", "before_files": [{"content": "import gzip\nimport logging\nimport os\nimport tarfile\nimport zipfile\nfrom functools import partial\nfrom urllib.request import urlretrieve\n\nimport torch\nfrom tqdm import tqdm\n\nfrom ..utils import reporthook\n\nlogger = logging.getLogger(__name__)\n\n\ndef _infer_shape(f):\n    num_lines, vector_dim = 0, None\n    for line in f:\n        if vector_dim is None:\n            row = line.rstrip().split(b\" \")\n            vector = row[1:]\n            # Assuming word, [vector] format\n            if len(vector) > 2:\n                # The header present in some (w2v) formats contains two elements.\n                vector_dim = len(vector)\n                num_lines += 1  # First element read\n        else:\n            num_lines += 1\n    f.seek(0)\n    return num_lines, vector_dim\n\n\nclass Vectors(object):\n    def __init__(self, name, cache=None, url=None, unk_init=None, max_vectors=None) -> None:\n        \"\"\"\n        Args:\n\n            name: name of the file that contains the vectors\n            cache: directory for cached vectors\n            url: url for download if vectors not found in cache\n            unk_init (callback): by default, initialize out-of-vocabulary word vectors\n                to zero vectors; can be any function that takes in a Tensor and returns a Tensor of the same size\n            max_vectors (int): this can be used to limit the number of\n                pre-trained vectors loaded.\n                Most pre-trained vector sets are sorted\n                in the descending order of word frequency.\n                Thus, in situations where the entire set doesn't fit in memory,\n                or is not needed for another reason, passing `max_vectors`\n                can limit the size of the loaded set.\n        \"\"\"\n\n        cache = \".vector_cache\" if cache is None else cache\n        self.itos = None\n        self.stoi = None\n        self.vectors = None\n        self.dim = None\n        self.unk_init = torch.Tensor.zero_ if unk_init is None else unk_init\n        self.cache(name, cache, url=url, max_vectors=max_vectors)\n\n    def __getitem__(self, token):\n        if token in self.stoi:\n            return self.vectors[self.stoi[token]]\n        else:\n            return self.unk_init(torch.Tensor(self.dim))\n\n    def cache(self, name, cache, url=None, max_vectors=None):\n        import ssl\n\n        ssl._create_default_https_context = ssl._create_unverified_context\n        if os.path.isfile(name):\n            path = name\n            if max_vectors:\n                file_suffix = \"_{}.pt\".format(max_vectors)\n            else:\n                file_suffix = \".pt\"\n            path_pt = os.path.join(cache, os.path.basename(name)) + file_suffix\n        else:\n            path = os.path.join(cache, name)\n            if max_vectors:\n                file_suffix = \"_{}.pt\".format(max_vectors)\n            else:\n                file_suffix = \".pt\"\n            path_pt = path + file_suffix\n\n        if not os.path.isfile(path_pt):\n            if not os.path.isfile(path) and url:\n                logger.info(\"Downloading vectors from {}\".format(url))\n                if not os.path.exists(cache):\n                    os.makedirs(cache)\n                dest = os.path.join(cache, os.path.basename(url))\n                if not os.path.isfile(dest):\n                    with tqdm(unit=\"B\", unit_scale=True, miniters=1, desc=dest) as t:\n                        try:\n                            urlretrieve(url, dest, reporthook=reporthook(t))\n                        except KeyboardInterrupt as e:  # remove the partial zip file\n                            os.remove(dest)\n                            raise e\n                logger.info(\"Extracting vectors into {}\".format(cache))\n                ext = os.path.splitext(dest)[1][1:]\n                if ext == \"zip\":\n                    with zipfile.ZipFile(dest, \"r\") as zf:\n                        zf.extractall(cache)\n                elif ext == \"gz\":\n                    if dest.endswith(\".tar.gz\"):\n                        with tarfile.open(dest, \"r:gz\") as tar:\n                            tar.extractall(path=cache)\n            if not os.path.isfile(path):\n                raise RuntimeError(\"no vectors found at {}\".format(path))\n\n            logger.info(\"Loading vectors from {}\".format(path))\n            ext = os.path.splitext(path)[1][1:]\n            if ext == \"gz\":\n                open_file = gzip.open\n            else:\n                open_file = open\n\n            vectors_loaded = 0\n            with open_file(path, \"rb\") as f:\n                num_lines, dim = _infer_shape(f)\n                if not max_vectors or max_vectors > num_lines:\n                    max_vectors = num_lines\n\n                itos, vectors, dim = [], torch.zeros((max_vectors, dim)), None\n\n                for line in tqdm(f, total=max_vectors):\n                    # Explicitly splitting on \" \" is important, so we don't\n                    # get rid of Unicode non-breaking spaces in the vectors.\n                    entries = line.rstrip().split(b\" \")\n\n                    word, entries = entries[0], entries[1:]\n                    if dim is None and len(entries) > 1:\n                        dim = len(entries)\n                    elif len(entries) == 1:\n                        logger.warning(\n                            \"Skipping token {} with 1-dimensional \" \"vector {}; likely a header\".format(word, entries)\n                        )\n                        continue\n                    elif dim != len(entries):\n                        raise RuntimeError(\n                            \"Vector for token {} has {} dimensions, but previously \"\n                            \"read vectors have {} dimensions. All vectors must have \"\n                            \"the same number of dimensions.\".format(word, len(entries), dim)\n                        )\n\n                    try:\n                        if isinstance(word, bytes):\n                            word = word.decode(\"utf-8\")\n                    except UnicodeDecodeError:\n                        logger.info(\"Skipping non-UTF8 token {}\".format(repr(word)))\n                        continue\n\n                    vectors[vectors_loaded] = torch.tensor([float(x) for x in entries])\n                    vectors_loaded += 1\n                    itos.append(word)\n\n                    if vectors_loaded == max_vectors:\n                        break\n\n            self.itos = itos\n            self.stoi = {word: i for i, word in enumerate(itos)}\n            self.vectors = torch.Tensor(vectors).view(-1, dim)\n            self.dim = dim\n            logger.info(\"Saving vectors to {}\".format(path_pt))\n            if not os.path.exists(cache):\n                os.makedirs(cache)\n            torch.save((self.itos, self.stoi, self.vectors, self.dim), path_pt)\n        else:\n            logger.info(\"Loading vectors from {}\".format(path_pt))\n            self.itos, self.stoi, self.vectors, self.dim = torch.load(path_pt)\n\n    def __len__(self):\n        return len(self.vectors)\n\n    def get_vecs_by_tokens(self, tokens, lower_case_backup=False):\n        \"\"\"Look up embedding vectors of tokens.\n\n        Args:\n            tokens: a token or a list of tokens. if `tokens` is a string,\n                returns a 1-D tensor of shape `self.dim`; if `tokens` is a\n                list of strings, returns a 2-D tensor of shape=(len(tokens),\n                self.dim).\n            lower_case_backup : Whether to look up the token in the lower case.\n                If False, each token in the original case will be looked up;\n                if True, each token in the original case will be looked up first,\n                if not found in the keys of the property `stoi`, the token in the\n                lower case will be looked up. Default: False.\n\n        Examples:\n            >>> examples = ['chip', 'baby', 'Beautiful']\n            >>> vec = text.vocab.GloVe(name='6B', dim=50)\n            >>> ret = vec.get_vecs_by_tokens(examples, lower_case_backup=True)\n        \"\"\"\n        to_reduce = False\n\n        if not isinstance(tokens, list):\n            tokens = [tokens]\n            to_reduce = True\n\n        if not lower_case_backup:\n            indices = [self[token] for token in tokens]\n        else:\n            indices = [self[token] if token in self.stoi else self[token.lower()] for token in tokens]\n\n        vecs = torch.stack(indices)\n        return vecs[0] if to_reduce else vecs\n\n\nclass GloVe(Vectors):\n    url = {\n        \"42B\": \"http://nlp.stanford.edu/data/glove.42B.300d.zip\",\n        \"840B\": \"http://nlp.stanford.edu/data/glove.840B.300d.zip\",\n        \"twitter.27B\": \"http://nlp.stanford.edu/data/glove.twitter.27B.zip\",\n        \"6B\": \"http://nlp.stanford.edu/data/glove.6B.zip\",\n    }\n\n    def __init__(self, name=\"840B\", dim=300, **kwargs) -> None:\n        url = self.url[name]\n        name = \"glove.{}.{}d.txt\".format(name, str(dim))\n        super(GloVe, self).__init__(name, url=url, **kwargs)\n\n\nclass FastText(Vectors):\n\n    url_base = \"https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.{}.vec\"\n\n    def __init__(self, language=\"en\", **kwargs) -> None:\n        url = self.url_base.format(language)\n        name = os.path.basename(url)\n        super(FastText, self).__init__(name, url=url, **kwargs)\n\n\nclass CharNGram(Vectors):\n\n    name = \"charNgram.txt\"\n    url = \"http://www.logos.t.u-tokyo.ac.jp/~hassy/publications/arxiv2016jmt/\" \"jmt_pre-trained_embeddings.tar.gz\"\n\n    def __init__(self, **kwargs) -> None:\n        super(CharNGram, self).__init__(self.name, url=self.url, **kwargs)\n\n    def __getitem__(self, token):\n        vector = torch.Tensor(1, self.dim).zero_()\n        if token == \"<unk>\":\n            return self.unk_init(vector)\n        chars = [\"#BEGIN#\"] + list(token) + [\"#END#\"]\n        num_vectors = 0\n        for n in [2, 3, 4]:\n            end = len(chars) - n + 1\n            grams = [chars[i : (i + n)] for i in range(end)]\n            for gram in grams:\n                gram_key = \"{}gram-{}\".format(n, \"\".join(gram))\n                if gram_key in self.stoi:\n                    vector += self.vectors[self.stoi[gram_key]]\n                    num_vectors += 1\n        if num_vectors > 0:\n            vector /= num_vectors\n        else:\n            vector = self.unk_init(vector)\n        return vector\n\n\npretrained_aliases = {\n    \"charngram.100d\": partial(CharNGram),\n    \"fasttext.en.300d\": partial(FastText, language=\"en\"),\n    \"fasttext.simple.300d\": partial(FastText, language=\"simple\"),\n    \"glove.42B.300d\": partial(GloVe, name=\"42B\", dim=\"300\"),\n    \"glove.840B.300d\": partial(GloVe, name=\"840B\", dim=\"300\"),\n    \"glove.twitter.27B.25d\": partial(GloVe, name=\"twitter.27B\", dim=\"25\"),\n    \"glove.twitter.27B.50d\": partial(GloVe, name=\"twitter.27B\", dim=\"50\"),\n    \"glove.twitter.27B.100d\": partial(GloVe, name=\"twitter.27B\", dim=\"100\"),\n    \"glove.twitter.27B.200d\": partial(GloVe, name=\"twitter.27B\", dim=\"200\"),\n    \"glove.6B.50d\": partial(GloVe, name=\"6B\", dim=\"50\"),\n    \"glove.6B.100d\": partial(GloVe, name=\"6B\", dim=\"100\"),\n    \"glove.6B.200d\": partial(GloVe, name=\"6B\", dim=\"200\"),\n    \"glove.6B.300d\": partial(GloVe, name=\"6B\", dim=\"300\"),\n}\n\"\"\"Mapping from string name to factory function\"\"\"\n", "path": "torchtext/vocab/vectors.py"}], "after_files": [{"content": "import gzip\nimport logging\nimport os\nimport tarfile\nimport zipfile\nfrom functools import partial\nfrom urllib.request import urlretrieve\n\nimport torch\nfrom tqdm import tqdm\n\nfrom ..utils import reporthook\n\nlogger = logging.getLogger(__name__)\n\n\ndef _infer_shape(f):\n    num_lines, vector_dim = 0, None\n    for line in f:\n        if vector_dim is None:\n            row = line.rstrip().split(b\" \")\n            vector = row[1:]\n            # Assuming word, [vector] format\n            if len(vector) > 2:\n                # The header present in some (w2v) formats contains two elements.\n                vector_dim = len(vector)\n                num_lines += 1  # First element read\n        else:\n            num_lines += 1\n    f.seek(0)\n    return num_lines, vector_dim\n\n\nclass Vectors(object):\n    def __init__(self, name, cache=None, url=None, unk_init=None, max_vectors=None) -> None:\n        \"\"\"\n        Args:\n\n            name: name of the file that contains the vectors\n            cache: directory for cached vectors\n            url: url for download if vectors not found in cache\n            unk_init (callback): by default, initialize out-of-vocabulary word vectors\n                to zero vectors; can be any function that takes in a Tensor and returns a Tensor of the same size\n            max_vectors (int): this can be used to limit the number of\n                pre-trained vectors loaded.\n                Most pre-trained vector sets are sorted\n                in the descending order of word frequency.\n                Thus, in situations where the entire set doesn't fit in memory,\n                or is not needed for another reason, passing `max_vectors`\n                can limit the size of the loaded set.\n        \"\"\"\n\n        cache = \".vector_cache\" if cache is None else cache\n        self.itos = None\n        self.stoi = None\n        self.vectors = None\n        self.dim = None\n        self.unk_init = torch.Tensor.zero_ if unk_init is None else unk_init\n        self.cache(name, cache, url=url, max_vectors=max_vectors)\n\n    def __getitem__(self, token):\n        if token in self.stoi:\n            return self.vectors[self.stoi[token]]\n        else:\n            return self.unk_init(torch.Tensor(self.dim))\n\n    def __contains__(self, token):\n        return token in self.stoi\n\n    def cache(self, name, cache, url=None, max_vectors=None):\n        import ssl\n\n        ssl._create_default_https_context = ssl._create_unverified_context\n        if os.path.isfile(name):\n            path = name\n            if max_vectors:\n                file_suffix = \"_{}.pt\".format(max_vectors)\n            else:\n                file_suffix = \".pt\"\n            path_pt = os.path.join(cache, os.path.basename(name)) + file_suffix\n        else:\n            path = os.path.join(cache, name)\n            if max_vectors:\n                file_suffix = \"_{}.pt\".format(max_vectors)\n            else:\n                file_suffix = \".pt\"\n            path_pt = path + file_suffix\n\n        if not os.path.isfile(path_pt):\n            if not os.path.isfile(path) and url:\n                logger.info(\"Downloading vectors from {}\".format(url))\n                if not os.path.exists(cache):\n                    os.makedirs(cache)\n                dest = os.path.join(cache, os.path.basename(url))\n                if not os.path.isfile(dest):\n                    with tqdm(unit=\"B\", unit_scale=True, miniters=1, desc=dest) as t:\n                        try:\n                            urlretrieve(url, dest, reporthook=reporthook(t))\n                        except KeyboardInterrupt as e:  # remove the partial zip file\n                            os.remove(dest)\n                            raise e\n                logger.info(\"Extracting vectors into {}\".format(cache))\n                ext = os.path.splitext(dest)[1][1:]\n                if ext == \"zip\":\n                    with zipfile.ZipFile(dest, \"r\") as zf:\n                        zf.extractall(cache)\n                elif ext == \"gz\":\n                    if dest.endswith(\".tar.gz\"):\n                        with tarfile.open(dest, \"r:gz\") as tar:\n                            tar.extractall(path=cache)\n            if not os.path.isfile(path):\n                raise RuntimeError(\"no vectors found at {}\".format(path))\n\n            logger.info(\"Loading vectors from {}\".format(path))\n            ext = os.path.splitext(path)[1][1:]\n            if ext == \"gz\":\n                open_file = gzip.open\n            else:\n                open_file = open\n\n            vectors_loaded = 0\n            with open_file(path, \"rb\") as f:\n                num_lines, dim = _infer_shape(f)\n                if not max_vectors or max_vectors > num_lines:\n                    max_vectors = num_lines\n\n                itos, vectors, dim = [], torch.zeros((max_vectors, dim)), None\n\n                for line in tqdm(f, total=max_vectors):\n                    # Explicitly splitting on \" \" is important, so we don't\n                    # get rid of Unicode non-breaking spaces in the vectors.\n                    entries = line.rstrip().split(b\" \")\n\n                    word, entries = entries[0], entries[1:]\n                    if dim is None and len(entries) > 1:\n                        dim = len(entries)\n                    elif len(entries) == 1:\n                        logger.warning(\n                            \"Skipping token {} with 1-dimensional \" \"vector {}; likely a header\".format(word, entries)\n                        )\n                        continue\n                    elif dim != len(entries):\n                        raise RuntimeError(\n                            \"Vector for token {} has {} dimensions, but previously \"\n                            \"read vectors have {} dimensions. All vectors must have \"\n                            \"the same number of dimensions.\".format(word, len(entries), dim)\n                        )\n\n                    try:\n                        if isinstance(word, bytes):\n                            word = word.decode(\"utf-8\")\n                    except UnicodeDecodeError:\n                        logger.info(\"Skipping non-UTF8 token {}\".format(repr(word)))\n                        continue\n\n                    vectors[vectors_loaded] = torch.tensor([float(x) for x in entries])\n                    vectors_loaded += 1\n                    itos.append(word)\n\n                    if vectors_loaded == max_vectors:\n                        break\n\n            self.itos = itos\n            self.stoi = {word: i for i, word in enumerate(itos)}\n            self.vectors = torch.Tensor(vectors).view(-1, dim)\n            self.dim = dim\n            logger.info(\"Saving vectors to {}\".format(path_pt))\n            if not os.path.exists(cache):\n                os.makedirs(cache)\n            torch.save((self.itos, self.stoi, self.vectors, self.dim), path_pt)\n        else:\n            logger.info(\"Loading vectors from {}\".format(path_pt))\n            self.itos, self.stoi, self.vectors, self.dim = torch.load(path_pt)\n\n    def __len__(self):\n        return len(self.vectors)\n\n    def get_vecs_by_tokens(self, tokens, lower_case_backup=False):\n        \"\"\"Look up embedding vectors of tokens.\n\n        Args:\n            tokens: a token or a list of tokens. if `tokens` is a string,\n                returns a 1-D tensor of shape `self.dim`; if `tokens` is a\n                list of strings, returns a 2-D tensor of shape=(len(tokens),\n                self.dim).\n            lower_case_backup : Whether to look up the token in the lower case.\n                If False, each token in the original case will be looked up;\n                if True, each token in the original case will be looked up first,\n                if not found in the keys of the property `stoi`, the token in the\n                lower case will be looked up. Default: False.\n\n        Examples:\n            >>> examples = ['chip', 'baby', 'Beautiful']\n            >>> vec = text.vocab.GloVe(name='6B', dim=50)\n            >>> ret = vec.get_vecs_by_tokens(examples, lower_case_backup=True)\n        \"\"\"\n        to_reduce = False\n\n        if not isinstance(tokens, list):\n            tokens = [tokens]\n            to_reduce = True\n\n        if not lower_case_backup:\n            indices = [self[token] for token in tokens]\n        else:\n            indices = [self[token] if token in self.stoi else self[token.lower()] for token in tokens]\n\n        vecs = torch.stack(indices)\n        return vecs[0] if to_reduce else vecs\n\n\nclass GloVe(Vectors):\n    url = {\n        \"42B\": \"http://nlp.stanford.edu/data/glove.42B.300d.zip\",\n        \"840B\": \"http://nlp.stanford.edu/data/glove.840B.300d.zip\",\n        \"twitter.27B\": \"http://nlp.stanford.edu/data/glove.twitter.27B.zip\",\n        \"6B\": \"http://nlp.stanford.edu/data/glove.6B.zip\",\n    }\n\n    def __init__(self, name=\"840B\", dim=300, **kwargs) -> None:\n        url = self.url[name]\n        name = \"glove.{}.{}d.txt\".format(name, str(dim))\n        super(GloVe, self).__init__(name, url=url, **kwargs)\n\n\nclass FastText(Vectors):\n\n    url_base = \"https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.{}.vec\"\n\n    def __init__(self, language=\"en\", **kwargs) -> None:\n        url = self.url_base.format(language)\n        name = os.path.basename(url)\n        super(FastText, self).__init__(name, url=url, **kwargs)\n\n\nclass CharNGram(Vectors):\n\n    name = \"charNgram.txt\"\n    url = \"http://www.logos.t.u-tokyo.ac.jp/~hassy/publications/arxiv2016jmt/\" \"jmt_pre-trained_embeddings.tar.gz\"\n\n    def __init__(self, **kwargs) -> None:\n        super(CharNGram, self).__init__(self.name, url=self.url, **kwargs)\n\n    def __getitem__(self, token):\n        vector = torch.Tensor(1, self.dim).zero_()\n        if token == \"<unk>\":\n            return self.unk_init(vector)\n        chars = [\"#BEGIN#\"] + list(token) + [\"#END#\"]\n        num_vectors = 0\n        for n in [2, 3, 4]:\n            end = len(chars) - n + 1\n            grams = [chars[i : (i + n)] for i in range(end)]\n            for gram in grams:\n                gram_key = \"{}gram-{}\".format(n, \"\".join(gram))\n                if gram_key in self.stoi:\n                    vector += self.vectors[self.stoi[gram_key]]\n                    num_vectors += 1\n        if num_vectors > 0:\n            vector /= num_vectors\n        else:\n            vector = self.unk_init(vector)\n        return vector\n\n\npretrained_aliases = {\n    \"charngram.100d\": partial(CharNGram),\n    \"fasttext.en.300d\": partial(FastText, language=\"en\"),\n    \"fasttext.simple.300d\": partial(FastText, language=\"simple\"),\n    \"glove.42B.300d\": partial(GloVe, name=\"42B\", dim=\"300\"),\n    \"glove.840B.300d\": partial(GloVe, name=\"840B\", dim=\"300\"),\n    \"glove.twitter.27B.25d\": partial(GloVe, name=\"twitter.27B\", dim=\"25\"),\n    \"glove.twitter.27B.50d\": partial(GloVe, name=\"twitter.27B\", dim=\"50\"),\n    \"glove.twitter.27B.100d\": partial(GloVe, name=\"twitter.27B\", dim=\"100\"),\n    \"glove.twitter.27B.200d\": partial(GloVe, name=\"twitter.27B\", dim=\"200\"),\n    \"glove.6B.50d\": partial(GloVe, name=\"6B\", dim=\"50\"),\n    \"glove.6B.100d\": partial(GloVe, name=\"6B\", dim=\"100\"),\n    \"glove.6B.200d\": partial(GloVe, name=\"6B\", dim=\"200\"),\n    \"glove.6B.300d\": partial(GloVe, name=\"6B\", dim=\"300\"),\n}\n\"\"\"Mapping from string name to factory function\"\"\"\n", "path": "torchtext/vocab/vectors.py"}]}