prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Add a `max_words` argument to `build_vocab_from_iterator`
  ## 🚀 Feature
  <!-- A clear and concise description of the feature proposal -->
  
  [Link to the docs](https://pytorch.org/text/stable/vocab.html?highlight=build%20vocab#torchtext.vocab.build_vocab_from_iterator)
  
  I believe it would be beneficial to limit the number of words you want in your vocabulary with an argument like `max_words`, e.g.:
  ```
  vocab = build_vocab_from_iterator(yield_tokens_batch(file_path), specials=["<unk>"], max_words=50000)
  ```
  
  **Motivation**
  
  <!-- Please outline the motivation for the proposal. Is your feature request related to a problem? e.g., I'm always frustrated when [...]. If this is related to another GitHub issue, please link here too -->
  
  
  This allows a controllable-sized `nn.Embedding`, with rare words being mapped to `<unk>`. Otherwise, it would not be practical to use `build_vocab_from_iterator` for larger datasets.
  
  
  **Alternatives**
  
  <!-- A clear and concise description of any alternative solutions or features you've considered, if any. -->
  
  Keras and Huggingface's tokenizers would be viable alternatives, but do not nicely integrate with the torchtext ecosystem.
  
  
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `torchtext/vocab/vocab_factory.py`
  Content:
  ```
  1 from .vocab import Vocab
  2 from typing import Dict, Iterable, Optional, List
  3 from collections import Counter, OrderedDict
  4 from torchtext._torchtext import (
  5     Vocab as VocabPybind,
  6 )
  7 
  8 
  9 def vocab(ordered_dict: Dict, min_freq: int = 1,
  10           specials: Optional[List[str]] = None,
  11           special_first: bool = True) -> Vocab:
  12     r"""Factory method for creating a vocab object which maps tokens to indices.
  13 
  14     Note that the ordering in which key value pairs were inserted in the `ordered_dict` will be respected when building the vocab.
  15     Therefore if sorting by token frequency is important to the user, the `ordered_dict` should be created in a way to reflect this.
  16 
  17     Args:
  18         ordered_dict: Ordered Dictionary mapping tokens to their corresponding occurance frequencies.
  19         min_freq: The minimum frequency needed to include a token in the vocabulary.
  20         specials: Special symbols to add. The order of supplied tokens will be preserved.
  21         special_first: Indicates whether to insert symbols at the beginning or at the end.
  22 
  23     Returns:
  24         torchtext.vocab.Vocab: A `Vocab` object
  25 
  26     Examples:
  27         >>> from torchtext.vocab import vocab
  28         >>> from collections import Counter, OrderedDict
  29         >>> counter = Counter(["a", "a", "b", "b", "b"])
  30         >>> sorted_by_freq_tuples = sorted(counter.items(), key=lambda x: x[1], reverse=True)
  31         >>> ordered_dict = OrderedDict(sorted_by_freq_tuples)
  32         >>> v1 = vocab(ordered_dict)
  33         >>> print(v1['a']) #prints 1
  34         >>> print(v1['out of vocab']) #raise RuntimeError since default index is not set
  35         >>> tokens = ['e', 'd', 'c', 'b', 'a']
  36         >>> #adding <unk> token and default index
  37         >>> unk_token = '<unk>'
  38         >>> default_index = -1
  39         >>> v2 = vocab(OrderedDict([(token, 1) for token in tokens]), specials=[unk_token])
  40         >>> v2.set_default_index(default_index)
  41         >>> print(v2['<unk>']) #prints 0
  42         >>> print(v2['out of vocab']) #prints -1
  43         >>> #make default index same as index of unk_token
  44         >>> v2.set_default_index(v2[unk_token])
  45         >>> v2['out of vocab'] is v2[unk_token] #prints True
  46     """
  47     specials = specials or []
  48     for token in specials:
  49         ordered_dict.pop(token, None)
  50 
  51     tokens = []
  52     for token, freq in ordered_dict.items():
  53         if freq >= min_freq:
  54             tokens.append(token)
  55 
  56     if special_first:
  57         tokens[0:0] = specials
  58     else:
  59         tokens.extend(specials)
  60 
  61     return Vocab(VocabPybind(tokens, None))
  62 
  63 
  64 def build_vocab_from_iterator(iterator: Iterable, min_freq: int = 1, specials: Optional[List[str]] = None, special_first: bool = True) -> Vocab:
  65     """
  66     Build a Vocab from an iterator.
  67 
  68     Args:
  69         iterator: Iterator used to build Vocab. Must yield list or iterator of tokens.
  70         min_freq: The minimum frequency needed to include a token in the vocabulary.
  71         specials: Special symbols to add. The order of supplied tokens will be preserved.
  72         special_first: Indicates whether to insert symbols at the beginning or at the end.
  73 
  74 
  75     Returns:
  76         torchtext.vocab.Vocab: A `Vocab` object
  77 
  78     Examples:
  79         >>> #generating vocab from text file
  80         >>> import io
  81         >>> from torchtext.vocab import build_vocab_from_iterator
  82         >>> def yield_tokens(file_path):
  83         >>>     with io.open(file_path, encoding = 'utf-8') as f:
  84         >>>         for line in f:
  85         >>>             yield line.strip().split()
  86         >>> vocab = build_vocab_from_iterator(yield_tokens_batch(file_path), specials=["<unk>"])
  87     """
  88 
  89     counter = Counter()
  90     for tokens in iterator:
  91         counter.update(tokens)
  92 
  93     sorted_by_freq_tuples = sorted(counter.items(), key=lambda x: x[0])
  94     sorted_by_freq_tuples.sort(key=lambda x: x[1], reverse=True)
  95     ordered_dict = OrderedDict(sorted_by_freq_tuples)
  96 
  97     word_vocab = vocab(ordered_dict, min_freq=min_freq, specials=specials or [],
  98                        special_first=special_first)
  99     return word_vocab
  100
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/torchtext/vocab/vocab_factory.py b/torchtext/vocab/vocab_factory.py
  --- a/torchtext/vocab/vocab_factory.py
  +++ b/torchtext/vocab/vocab_factory.py
  @@ -49,6 +49,7 @@
           ordered_dict.pop(token, None)
   
       tokens = []
  +    # Save room for special tokens
       for token, freq in ordered_dict.items():
           if freq >= min_freq:
               tokens.append(token)
  @@ -61,7 +62,7 @@
       return Vocab(VocabPybind(tokens, None))
   
   
  -def build_vocab_from_iterator(iterator: Iterable, min_freq: int = 1, specials: Optional[List[str]] = None, special_first: bool = True) -> Vocab:
  +def build_vocab_from_iterator(iterator: Iterable, min_freq: int = 1, specials: Optional[List[str]] = None, special_first: bool = True, max_tokens: Optional[int] = None) -> Vocab:
       """
       Build a Vocab from an iterator.
   
  @@ -70,6 +71,7 @@
           min_freq: The minimum frequency needed to include a token in the vocabulary.
           specials: Special symbols to add. The order of supplied tokens will be preserved.
           special_first: Indicates whether to insert symbols at the beginning or at the end.
  +        max_tokens: If provided, creates the vocab from the `max_tokens - len(specials)` most frequent tokens.
   
   
       Returns:
  @@ -90,10 +92,16 @@
       for tokens in iterator:
           counter.update(tokens)
   
  -    sorted_by_freq_tuples = sorted(counter.items(), key=lambda x: x[0])
  -    sorted_by_freq_tuples.sort(key=lambda x: x[1], reverse=True)
  -    ordered_dict = OrderedDict(sorted_by_freq_tuples)
  +    specials = specials or []
  +
  +    # First sort by descending frequency, then lexicographically
  +    sorted_by_freq_tuples = sorted(counter.items(), key=lambda x: (-x[1], x[0]))
  +
  +    if max_tokens is None:
  +        ordered_dict = OrderedDict(sorted_by_freq_tuples)
  +    else:
  +        assert len(specials) < max_tokens, "len(specials) >= max_tokens, so the vocab will be entirely special tokens."
  +        ordered_dict = OrderedDict(sorted_by_freq_tuples[:max_tokens - len(specials)])
   
  -    word_vocab = vocab(ordered_dict, min_freq=min_freq, specials=specials or [],
  -                       special_first=special_first)
  +    word_vocab = vocab(ordered_dict, min_freq=min_freq, specials=specials, special_first=special_first)
       return word_vocab
  
verification_info: {"golden_diff": "diff --git a/torchtext/vocab/vocab_factory.py b/torchtext/vocab/vocab_factory.py\n--- a/torchtext/vocab/vocab_factory.py\n+++ b/torchtext/vocab/vocab_factory.py\n@@ -49,6 +49,7 @@\n         ordered_dict.pop(token, None)\n \n     tokens = []\n+    # Save room for special tokens\n     for token, freq in ordered_dict.items():\n         if freq >= min_freq:\n             tokens.append(token)\n@@ -61,7 +62,7 @@\n     return Vocab(VocabPybind(tokens, None))\n \n \n-def build_vocab_from_iterator(iterator: Iterable, min_freq: int = 1, specials: Optional[List[str]] = None, special_first: bool = True) -> Vocab:\n+def build_vocab_from_iterator(iterator: Iterable, min_freq: int = 1, specials: Optional[List[str]] = None, special_first: bool = True, max_tokens: Optional[int] = None) -> Vocab:\n     \"\"\"\n     Build a Vocab from an iterator.\n \n@@ -70,6 +71,7 @@\n         min_freq: The minimum frequency needed to include a token in the vocabulary.\n         specials: Special symbols to add. The order of supplied tokens will be preserved.\n         special_first: Indicates whether to insert symbols at the beginning or at the end.\n+        max_tokens: If provided, creates the vocab from the `max_tokens - len(specials)` most frequent tokens.\n \n \n     Returns:\n@@ -90,10 +92,16 @@\n     for tokens in iterator:\n         counter.update(tokens)\n \n-    sorted_by_freq_tuples = sorted(counter.items(), key=lambda x: x[0])\n-    sorted_by_freq_tuples.sort(key=lambda x: x[1], reverse=True)\n-    ordered_dict = OrderedDict(sorted_by_freq_tuples)\n+    specials = specials or []\n+\n+    # First sort by descending frequency, then lexicographically\n+    sorted_by_freq_tuples = sorted(counter.items(), key=lambda x: (-x[1], x[0]))\n+\n+    if max_tokens is None:\n+        ordered_dict = OrderedDict(sorted_by_freq_tuples)\n+    else:\n+        assert len(specials) < max_tokens, \"len(specials) >= max_tokens, so the vocab will be entirely special tokens.\"\n+        ordered_dict = OrderedDict(sorted_by_freq_tuples[:max_tokens - len(specials)])\n \n-    word_vocab = vocab(ordered_dict, min_freq=min_freq, specials=specials or [],\n-                       special_first=special_first)\n+    word_vocab = vocab(ordered_dict, min_freq=min_freq, specials=specials, special_first=special_first)\n     return word_vocab\n", "issue": "Add a `max_words` argument to `build_vocab_from_iterator`\n## \ud83d\ude80 Feature\r\n<!-- A clear and concise description of the feature proposal -->\r\n\r\n[Link to the docs](https://pytorch.org/text/stable/vocab.html?highlight=build%20vocab#torchtext.vocab.build_vocab_from_iterator)\r\n\r\nI believe it would be beneficial to limit the number of words you want in your vocabulary with an argument like `max_words`, e.g.:\r\n```\r\nvocab = build_vocab_from_iterator(yield_tokens_batch(file_path), specials=[\"<unk>\"], max_words=50000)\r\n```\r\n\r\n**Motivation**\r\n\r\n<!-- Please outline the motivation for the proposal. Is your feature request related to a problem? e.g., I'm always frustrated when [...]. If this is related to another GitHub issue, please link here too -->\r\n\r\n\r\nThis allows a controllable-sized `nn.Embedding`, with rare words being mapped to `<unk>`. Otherwise, it would not be practical to use `build_vocab_from_iterator` for larger datasets.\r\n\r\n\r\n**Alternatives**\r\n\r\n<!-- A clear and concise description of any alternative solutions or features you've considered, if any. -->\r\n\r\nKeras and Huggingface's tokenizers would be viable alternatives, but do not nicely integrate with the torchtext ecosystem.\r\n\r\n\n", "before_files": [{"content": "from .vocab import Vocab\nfrom typing import Dict, Iterable, Optional, List\nfrom collections import Counter, OrderedDict\nfrom torchtext._torchtext import (\n    Vocab as VocabPybind,\n)\n\n\ndef vocab(ordered_dict: Dict, min_freq: int = 1,\n          specials: Optional[List[str]] = None,\n          special_first: bool = True) -> Vocab:\n    r\"\"\"Factory method for creating a vocab object which maps tokens to indices.\n\n    Note that the ordering in which key value pairs were inserted in the `ordered_dict` will be respected when building the vocab.\n    Therefore if sorting by token frequency is important to the user, the `ordered_dict` should be created in a way to reflect this.\n\n    Args:\n        ordered_dict: Ordered Dictionary mapping tokens to their corresponding occurance frequencies.\n        min_freq: The minimum frequency needed to include a token in the vocabulary.\n        specials: Special symbols to add. The order of supplied tokens will be preserved.\n        special_first: Indicates whether to insert symbols at the beginning or at the end.\n\n    Returns:\n        torchtext.vocab.Vocab: A `Vocab` object\n\n    Examples:\n        >>> from torchtext.vocab import vocab\n        >>> from collections import Counter, OrderedDict\n        >>> counter = Counter([\"a\", \"a\", \"b\", \"b\", \"b\"])\n        >>> sorted_by_freq_tuples = sorted(counter.items(), key=lambda x: x[1], reverse=True)\n        >>> ordered_dict = OrderedDict(sorted_by_freq_tuples)\n        >>> v1 = vocab(ordered_dict)\n        >>> print(v1['a']) #prints 1\n        >>> print(v1['out of vocab']) #raise RuntimeError since default index is not set\n        >>> tokens = ['e', 'd', 'c', 'b', 'a']\n        >>> #adding <unk> token and default index\n        >>> unk_token = '<unk>'\n        >>> default_index = -1\n        >>> v2 = vocab(OrderedDict([(token, 1) for token in tokens]), specials=[unk_token])\n        >>> v2.set_default_index(default_index)\n        >>> print(v2['<unk>']) #prints 0\n        >>> print(v2['out of vocab']) #prints -1\n        >>> #make default index same as index of unk_token\n        >>> v2.set_default_index(v2[unk_token])\n        >>> v2['out of vocab'] is v2[unk_token] #prints True\n    \"\"\"\n    specials = specials or []\n    for token in specials:\n        ordered_dict.pop(token, None)\n\n    tokens = []\n    for token, freq in ordered_dict.items():\n        if freq >= min_freq:\n            tokens.append(token)\n\n    if special_first:\n        tokens[0:0] = specials\n    else:\n        tokens.extend(specials)\n\n    return Vocab(VocabPybind(tokens, None))\n\n\ndef build_vocab_from_iterator(iterator: Iterable, min_freq: int = 1, specials: Optional[List[str]] = None, special_first: bool = True) -> Vocab:\n    \"\"\"\n    Build a Vocab from an iterator.\n\n    Args:\n        iterator: Iterator used to build Vocab. Must yield list or iterator of tokens.\n        min_freq: The minimum frequency needed to include a token in the vocabulary.\n        specials: Special symbols to add. The order of supplied tokens will be preserved.\n        special_first: Indicates whether to insert symbols at the beginning or at the end.\n\n\n    Returns:\n        torchtext.vocab.Vocab: A `Vocab` object\n\n    Examples:\n        >>> #generating vocab from text file\n        >>> import io\n        >>> from torchtext.vocab import build_vocab_from_iterator\n        >>> def yield_tokens(file_path):\n        >>>     with io.open(file_path, encoding = 'utf-8') as f:\n        >>>         for line in f:\n        >>>             yield line.strip().split()\n        >>> vocab = build_vocab_from_iterator(yield_tokens_batch(file_path), specials=[\"<unk>\"])\n    \"\"\"\n\n    counter = Counter()\n    for tokens in iterator:\n        counter.update(tokens)\n\n    sorted_by_freq_tuples = sorted(counter.items(), key=lambda x: x[0])\n    sorted_by_freq_tuples.sort(key=lambda x: x[1], reverse=True)\n    ordered_dict = OrderedDict(sorted_by_freq_tuples)\n\n    word_vocab = vocab(ordered_dict, min_freq=min_freq, specials=specials or [],\n                       special_first=special_first)\n    return word_vocab\n", "path": "torchtext/vocab/vocab_factory.py"}], "after_files": [{"content": "from .vocab import Vocab\nfrom typing import Dict, Iterable, Optional, List\nfrom collections import Counter, OrderedDict\nfrom torchtext._torchtext import (\n    Vocab as VocabPybind,\n)\n\n\ndef vocab(ordered_dict: Dict, min_freq: int = 1,\n          specials: Optional[List[str]] = None,\n          special_first: bool = True) -> Vocab:\n    r\"\"\"Factory method for creating a vocab object which maps tokens to indices.\n\n    Note that the ordering in which key value pairs were inserted in the `ordered_dict` will be respected when building the vocab.\n    Therefore if sorting by token frequency is important to the user, the `ordered_dict` should be created in a way to reflect this.\n\n    Args:\n        ordered_dict: Ordered Dictionary mapping tokens to their corresponding occurance frequencies.\n        min_freq: The minimum frequency needed to include a token in the vocabulary.\n        specials: Special symbols to add. The order of supplied tokens will be preserved.\n        special_first: Indicates whether to insert symbols at the beginning or at the end.\n\n    Returns:\n        torchtext.vocab.Vocab: A `Vocab` object\n\n    Examples:\n        >>> from torchtext.vocab import vocab\n        >>> from collections import Counter, OrderedDict\n        >>> counter = Counter([\"a\", \"a\", \"b\", \"b\", \"b\"])\n        >>> sorted_by_freq_tuples = sorted(counter.items(), key=lambda x: x[1], reverse=True)\n        >>> ordered_dict = OrderedDict(sorted_by_freq_tuples)\n        >>> v1 = vocab(ordered_dict)\n        >>> print(v1['a']) #prints 1\n        >>> print(v1['out of vocab']) #raise RuntimeError since default index is not set\n        >>> tokens = ['e', 'd', 'c', 'b', 'a']\n        >>> #adding <unk> token and default index\n        >>> unk_token = '<unk>'\n        >>> default_index = -1\n        >>> v2 = vocab(OrderedDict([(token, 1) for token in tokens]), specials=[unk_token])\n        >>> v2.set_default_index(default_index)\n        >>> print(v2['<unk>']) #prints 0\n        >>> print(v2['out of vocab']) #prints -1\n        >>> #make default index same as index of unk_token\n        >>> v2.set_default_index(v2[unk_token])\n        >>> v2['out of vocab'] is v2[unk_token] #prints True\n    \"\"\"\n    specials = specials or []\n    for token in specials:\n        ordered_dict.pop(token, None)\n\n    tokens = []\n    # Save room for special tokens\n    for token, freq in ordered_dict.items():\n        if freq >= min_freq:\n            tokens.append(token)\n\n    if special_first:\n        tokens[0:0] = specials\n    else:\n        tokens.extend(specials)\n\n    return Vocab(VocabPybind(tokens, None))\n\n\ndef build_vocab_from_iterator(iterator: Iterable, min_freq: int = 1, specials: Optional[List[str]] = None, special_first: bool = True, max_tokens: Optional[int] = None) -> Vocab:\n    \"\"\"\n    Build a Vocab from an iterator.\n\n    Args:\n        iterator: Iterator used to build Vocab. Must yield list or iterator of tokens.\n        min_freq: The minimum frequency needed to include a token in the vocabulary.\n        specials: Special symbols to add. The order of supplied tokens will be preserved.\n        special_first: Indicates whether to insert symbols at the beginning or at the end.\n        max_tokens: If provided, creates the vocab from the `max_tokens - len(specials)` most frequent tokens.\n\n\n    Returns:\n        torchtext.vocab.Vocab: A `Vocab` object\n\n    Examples:\n        >>> #generating vocab from text file\n        >>> import io\n        >>> from torchtext.vocab import build_vocab_from_iterator\n        >>> def yield_tokens(file_path):\n        >>>     with io.open(file_path, encoding = 'utf-8') as f:\n        >>>         for line in f:\n        >>>             yield line.strip().split()\n        >>> vocab = build_vocab_from_iterator(yield_tokens_batch(file_path), specials=[\"<unk>\"])\n    \"\"\"\n\n    counter = Counter()\n    for tokens in iterator:\n        counter.update(tokens)\n\n    specials = specials or []\n\n    # First sort by descending frequency, then lexicographically\n    sorted_by_freq_tuples = sorted(counter.items(), key=lambda x: (-x[1], x[0]))\n\n    if max_tokens is None:\n        ordered_dict = OrderedDict(sorted_by_freq_tuples)\n    else:\n        assert len(specials) < max_tokens, \"len(specials) >= max_tokens, so the vocab will be entirely special tokens.\"\n        ordered_dict = OrderedDict(sorted_by_freq_tuples[:max_tokens - len(specials)])\n\n    word_vocab = vocab(ordered_dict, min_freq=min_freq, specials=specials, special_first=special_first)\n    return word_vocab\n", "path": "torchtext/vocab/vocab_factory.py"}]}