We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
AttributeError: 'itertools.chain' object has no attribute 'lookup'
nltk version: 3.7.
python version: 3.9.15
From the following code:

```
padded_line = [list(pad_both_ends(word_tokenize(input_text), n=N))]
train_data, vocab = padded_everygram_pipeline(order=N, text=padded_line)
lm = Lidstone(gamma=0.2, vocabulary=vocab, order=N)
lm.fit(text=train_data)
```

where input_text is a long string and N as in the Ngram model. From the last line of code above, I get the following error:

>lm.fit(text=train_data)
>  File "env\lib\site-packages\nltk\lm\api.py", line 110, in fit
>  self.counts.update(self.vocab.lookup(sent) for sent in text)
>  File "env\lib\site-packages\nltk\lm\counter.py", line 115, in update
>    for sent in ngram_text:
>  File "env\lib\site-packages\nltk\lm\api.py", line 110, in <genexpr>
>    self.counts.update(self.vocab.lookup(sent) for sent in text)
> AttributeError: 'itertools.chain' object has no attribute 'lookup'

Can anyone fix this error for me?

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `nltk/lm/api.py`
Content:
```
1 # Natural Language Toolkit: Language Models
2 #
3 # Copyright (C) 2001-2022 NLTK Project
4 # Authors: Ilia Kurenkov <ilia.kurenkov@gmail.com>
5 # URL: <https://www.nltk.org/>
6 # For license information, see LICENSE.TXT
7 """Language Model Interface."""
8 
9 import random
10 from abc import ABCMeta, abstractmethod
11 from bisect import bisect
12 from itertools import accumulate
13 
14 from nltk.lm.counter import NgramCounter
15 from nltk.lm.util import log_base2
16 from nltk.lm.vocabulary import Vocabulary
17 
18 
19 class Smoothing(metaclass=ABCMeta):
20     """Ngram Smoothing Interface
21 
22     Implements Chen & Goodman 1995's idea that all smoothing algorithms have
23     certain features in common. This should ideally allow smoothing algorithms to
24     work both with Backoff and Interpolation.
25     """
26 
27     def __init__(self, vocabulary, counter):
28         """
29         :param vocabulary: The Ngram vocabulary object.
30         :type vocabulary: nltk.lm.vocab.Vocabulary
31         :param counter: The counts of the vocabulary items.
32         :type counter: nltk.lm.counter.NgramCounter
33         """
34         self.vocab = vocabulary
35         self.counts = counter
36 
37     @abstractmethod
38     def unigram_score(self, word):
39         raise NotImplementedError()
40 
41     @abstractmethod
42     def alpha_gamma(self, word, context):
43         raise NotImplementedError()
44 
45 
46 def _mean(items):
47     """Return average (aka mean) for sequence of items."""
48     return sum(items) / len(items)
49 
50 
51 def _random_generator(seed_or_generator):
52     if isinstance(seed_or_generator, random.Random):
53         return seed_or_generator
54     return random.Random(seed_or_generator)
55 
56 
57 def _weighted_choice(population, weights, random_generator=None):
58     """Like random.choice, but with weights.
59 
60     Heavily inspired by python 3.6 `random.choices`.
61     """
62     if not population:
63         raise ValueError("Can't choose from empty population")
64     if len(population) != len(weights):
65         raise ValueError("The number of weights does not match the population")
66     cum_weights = list(accumulate(weights))
67     total = cum_weights[-1]
68     threshold = random_generator.random()
69     return population[bisect(cum_weights, total * threshold)]
70 
71 
72 class LanguageModel(metaclass=ABCMeta):
73     """ABC for Language Models.
74 
75     Cannot be directly instantiated itself.
76 
77     """
78 
79     def __init__(self, order, vocabulary=None, counter=None):
80         """Creates new LanguageModel.
81 
82         :param vocabulary: If provided, this vocabulary will be used instead
83             of creating a new one when training.
84         :type vocabulary: `nltk.lm.Vocabulary` or None
85         :param counter: If provided, use this object to count ngrams.
86         :type vocabulary: `nltk.lm.NgramCounter` or None
87         :param ngrams_fn: If given, defines how sentences in training text are turned to ngram
88             sequences.
89         :type ngrams_fn: function or None
90         :param pad_fn: If given, defines how sentences in training text are padded.
91         :type pad_fn: function or None
92         """
93         self.order = order
94         self.vocab = Vocabulary() if vocabulary is None else vocabulary
95         self.counts = NgramCounter() if counter is None else counter
96 
97     def fit(self, text, vocabulary_text=None):
98         """Trains the model on a text.
99 
100         :param text: Training text as a sequence of sentences.
101 
102         """
103         if not self.vocab:
104             if vocabulary_text is None:
105                 raise ValueError(
106                     "Cannot fit without a vocabulary or text to create it from."
107                 )
108             self.vocab.update(vocabulary_text)
109         self.counts.update(self.vocab.lookup(sent) for sent in text)
110 
111     def score(self, word, context=None):
112         """Masks out of vocab (OOV) words and computes their model score.
113 
114         For model-specific logic of calculating scores, see the `unmasked_score`
115         method.
116         """
117         return self.unmasked_score(
118             self.vocab.lookup(word), self.vocab.lookup(context) if context else None
119         )
120 
121     @abstractmethod
122     def unmasked_score(self, word, context=None):
123         """Score a word given some optional context.
124 
125         Concrete models are expected to provide an implementation.
126         Note that this method does not mask its arguments with the OOV label.
127         Use the `score` method for that.
128 
129         :param str word: Word for which we want the score
130         :param tuple(str) context: Context the word is in.
131             If `None`, compute unigram score.
132         :param context: tuple(str) or None
133         :rtype: float
134         """
135         raise NotImplementedError()
136 
137     def logscore(self, word, context=None):
138         """Evaluate the log score of this word in this context.
139 
140         The arguments are the same as for `score` and `unmasked_score`.
141 
142         """
143         return log_base2(self.score(word, context))
144 
145     def context_counts(self, context):
146         """Helper method for retrieving counts for a given context.
147 
148         Assumes context has been checked and oov words in it masked.
149         :type context: tuple(str) or None
150 
151         """
152         return (
153             self.counts[len(context) + 1][context] if context else self.counts.unigrams
154         )
155 
156     def entropy(self, text_ngrams):
157         """Calculate cross-entropy of model for given evaluation text.
158 
159         :param Iterable(tuple(str)) text_ngrams: A sequence of ngram tuples.
160         :rtype: float
161 
162         """
163         return -1 * _mean(
164             [self.logscore(ngram[-1], ngram[:-1]) for ngram in text_ngrams]
165         )
166 
167     def perplexity(self, text_ngrams):
168         """Calculates the perplexity of the given text.
169 
170         This is simply 2 ** cross-entropy for the text, so the arguments are the same.
171 
172         """
173         return pow(2.0, self.entropy(text_ngrams))
174 
175     def generate(self, num_words=1, text_seed=None, random_seed=None):
176         """Generate words from the model.
177 
178         :param int num_words: How many words to generate. By default 1.
179         :param text_seed: Generation can be conditioned on preceding context.
180         :param random_seed: A random seed or an instance of `random.Random`. If provided,
181             makes the random sampling part of generation reproducible.
182         :return: One (str) word or a list of words generated from model.
183 
184         Examples:
185 
186         >>> from nltk.lm import MLE
187         >>> lm = MLE(2)
188         >>> lm.fit([[("a", "b"), ("b", "c")]], vocabulary_text=['a', 'b', 'c'])
189         >>> lm.fit([[("a",), ("b",), ("c",)]])
190         >>> lm.generate(random_seed=3)
191         'a'
192         >>> lm.generate(text_seed=['a'])
193         'b'
194 
195         """
196         text_seed = [] if text_seed is None else list(text_seed)
197         random_generator = _random_generator(random_seed)
198         # This is the base recursion case.
199         if num_words == 1:
200             context = (
201                 text_seed[-self.order + 1 :]
202                 if len(text_seed) >= self.order
203                 else text_seed
204             )
205             samples = self.context_counts(self.vocab.lookup(context))
206             while context and not samples:
207                 context = context[1:] if len(context) > 1 else []
208                 samples = self.context_counts(self.vocab.lookup(context))
209             # Sorting samples achieves two things:
210             # - reproducible randomness when sampling
211             # - turns Mapping into Sequence which `_weighted_choice` expects
212             samples = sorted(samples)
213             return _weighted_choice(
214                 samples,
215                 tuple(self.score(w, context) for w in samples),
216                 random_generator,
217             )
218         # We build up text one word at a time using the preceding context.
219         generated = []
220         for _ in range(num_words):
221             generated.append(
222                 self.generate(
223                     num_words=1,
224                     text_seed=text_seed + generated,
225                     random_seed=random_generator,
226                 )
227             )
228         return generated
229
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/nltk/lm/api.py b/nltk/lm/api.py
--- a/nltk/lm/api.py
+++ b/nltk/lm/api.py
@@ -7,6 +7,7 @@
 """Language Model Interface."""
 
 import random
+import warnings
 from abc import ABCMeta, abstractmethod
 from bisect import bisect
 from itertools import accumulate
@@ -83,7 +84,7 @@
             of creating a new one when training.
         :type vocabulary: `nltk.lm.Vocabulary` or None
         :param counter: If provided, use this object to count ngrams.
-        :type vocabulary: `nltk.lm.NgramCounter` or None
+        :type counter: `nltk.lm.NgramCounter` or None
         :param ngrams_fn: If given, defines how sentences in training text are turned to ngram
             sequences.
         :type ngrams_fn: function or None
@@ -91,6 +92,12 @@
         :type pad_fn: function or None
         """
         self.order = order
+        if vocabulary and not isinstance(vocabulary, Vocabulary):
+            warnings.warn(
+                f"The `vocabulary` argument passed to {self.__class__.__name__!r} "
+                "must be an instance of `nltk.lm.Vocabulary`.",
+                stacklevel=3,
+            )
         self.vocab = Vocabulary() if vocabulary is None else vocabulary
         self.counts = NgramCounter() if counter is None else counter

{"golden_diff": "diff --git a/nltk/lm/api.py b/nltk/lm/api.py\n--- a/nltk/lm/api.py\n+++ b/nltk/lm/api.py\n@@ -7,6 +7,7 @@\n \"\"\"Language Model Interface.\"\"\"\n \n import random\n+import warnings\n from abc import ABCMeta, abstractmethod\n from bisect import bisect\n from itertools import accumulate\n@@ -83,7 +84,7 @@\n             of creating a new one when training.\n         :type vocabulary: `nltk.lm.Vocabulary` or None\n         :param counter: If provided, use this object to count ngrams.\n-        :type vocabulary: `nltk.lm.NgramCounter` or None\n+        :type counter: `nltk.lm.NgramCounter` or None\n         :param ngrams_fn: If given, defines how sentences in training text are turned to ngram\n             sequences.\n         :type ngrams_fn: function or None\n@@ -91,6 +92,12 @@\n         :type pad_fn: function or None\n         \"\"\"\n         self.order = order\n+        if vocabulary and not isinstance(vocabulary, Vocabulary):\n+            warnings.warn(\n+                f\"The `vocabulary` argument passed to {self.__class__.__name__!r} \"\n+                \"must be an instance of `nltk.lm.Vocabulary`.\",\n+                stacklevel=3,\n+            )\n         self.vocab = Vocabulary() if vocabulary is None else vocabulary\n         self.counts = NgramCounter() if counter is None else counter\n", "issue": "AttributeError: 'itertools.chain' object has no attribute 'lookup'\nnltk version: 3.7.\r\npython version: 3.9.15\r\nFrom the following code:\r\n\r\n```\r\npadded_line = [list(pad_both_ends(word_tokenize(input_text), n=N))]\r\ntrain_data, vocab = padded_everygram_pipeline(order=N, text=padded_line)\r\nlm = Lidstone(gamma=0.2, vocabulary=vocab, order=N)\r\nlm.fit(text=train_data)\r\n```\r\n\r\nwhere input_text is a long string and N as in the Ngram model. From the last line of code above, I get the following error:\r\n\r\n>lm.fit(text=train_data)\r\n>  File \"env\\lib\\site-packages\\nltk\\lm\\api.py\", line 110, in fit\r\n>  self.counts.update(self.vocab.lookup(sent) for sent in text)\r\n>  File \"env\\lib\\site-packages\\nltk\\lm\\counter.py\", line 115, in update\r\n>    for sent in ngram_text:\r\n>  File \"env\\lib\\site-packages\\nltk\\lm\\api.py\", line 110, in <genexpr>\r\n>    self.counts.update(self.vocab.lookup(sent) for sent in text)\r\n> AttributeError: 'itertools.chain' object has no attribute 'lookup'\r\n\r\nCan anyone fix this error for me?\n", "before_files": [{"content": "# Natural Language Toolkit: Language Models\n#\n# Copyright (C) 2001-2022 NLTK Project\n# Authors: Ilia Kurenkov <ilia.kurenkov@gmail.com>\n# URL: <https://www.nltk.org/>\n# For license information, see LICENSE.TXT\n\"\"\"Language Model Interface.\"\"\"\n\nimport random\nfrom abc import ABCMeta, abstractmethod\nfrom bisect import bisect\nfrom itertools import accumulate\n\nfrom nltk.lm.counter import NgramCounter\nfrom nltk.lm.util import log_base2\nfrom nltk.lm.vocabulary import Vocabulary\n\n\nclass Smoothing(metaclass=ABCMeta):\n    \"\"\"Ngram Smoothing Interface\n\n    Implements Chen & Goodman 1995's idea that all smoothing algorithms have\n    certain features in common. This should ideally allow smoothing algorithms to\n    work both with Backoff and Interpolation.\n    \"\"\"\n\n    def __init__(self, vocabulary, counter):\n        \"\"\"\n        :param vocabulary: The Ngram vocabulary object.\n        :type vocabulary: nltk.lm.vocab.Vocabulary\n        :param counter: The counts of the vocabulary items.\n        :type counter: nltk.lm.counter.NgramCounter\n        \"\"\"\n        self.vocab = vocabulary\n        self.counts = counter\n\n    @abstractmethod\n    def unigram_score(self, word):\n        raise NotImplementedError()\n\n    @abstractmethod\n    def alpha_gamma(self, word, context):\n        raise NotImplementedError()\n\n\ndef _mean(items):\n    \"\"\"Return average (aka mean) for sequence of items.\"\"\"\n    return sum(items) / len(items)\n\n\ndef _random_generator(seed_or_generator):\n    if isinstance(seed_or_generator, random.Random):\n        return seed_or_generator\n    return random.Random(seed_or_generator)\n\n\ndef _weighted_choice(population, weights, random_generator=None):\n    \"\"\"Like random.choice, but with weights.\n\n    Heavily inspired by python 3.6 `random.choices`.\n    \"\"\"\n    if not population:\n        raise ValueError(\"Can't choose from empty population\")\n    if len(population) != len(weights):\n        raise ValueError(\"The number of weights does not match the population\")\n    cum_weights = list(accumulate(weights))\n    total = cum_weights[-1]\n    threshold = random_generator.random()\n    return population[bisect(cum_weights, total * threshold)]\n\n\nclass LanguageModel(metaclass=ABCMeta):\n    \"\"\"ABC for Language Models.\n\n    Cannot be directly instantiated itself.\n\n    \"\"\"\n\n    def __init__(self, order, vocabulary=None, counter=None):\n        \"\"\"Creates new LanguageModel.\n\n        :param vocabulary: If provided, this vocabulary will be used instead\n            of creating a new one when training.\n        :type vocabulary: `nltk.lm.Vocabulary` or None\n        :param counter: If provided, use this object to count ngrams.\n        :type vocabulary: `nltk.lm.NgramCounter` or None\n        :param ngrams_fn: If given, defines how sentences in training text are turned to ngram\n            sequences.\n        :type ngrams_fn: function or None\n        :param pad_fn: If given, defines how sentences in training text are padded.\n        :type pad_fn: function or None\n        \"\"\"\n        self.order = order\n        self.vocab = Vocabulary() if vocabulary is None else vocabulary\n        self.counts = NgramCounter() if counter is None else counter\n\n    def fit(self, text, vocabulary_text=None):\n        \"\"\"Trains the model on a text.\n\n        :param text: Training text as a sequence of sentences.\n\n        \"\"\"\n        if not self.vocab:\n            if vocabulary_text is None:\n                raise ValueError(\n                    \"Cannot fit without a vocabulary or text to create it from.\"\n                )\n            self.vocab.update(vocabulary_text)\n        self.counts.update(self.vocab.lookup(sent) for sent in text)\n\n    def score(self, word, context=None):\n        \"\"\"Masks out of vocab (OOV) words and computes their model score.\n\n        For model-specific logic of calculating scores, see the `unmasked_score`\n        method.\n        \"\"\"\n        return self.unmasked_score(\n            self.vocab.lookup(word), self.vocab.lookup(context) if context else None\n        )\n\n    @abstractmethod\n    def unmasked_score(self, word, context=None):\n        \"\"\"Score a word given some optional context.\n\n        Concrete models are expected to provide an implementation.\n        Note that this method does not mask its arguments with the OOV label.\n        Use the `score` method for that.\n\n        :param str word: Word for which we want the score\n        :param tuple(str) context: Context the word is in.\n            If `None`, compute unigram score.\n        :param context: tuple(str) or None\n        :rtype: float\n        \"\"\"\n        raise NotImplementedError()\n\n    def logscore(self, word, context=None):\n        \"\"\"Evaluate the log score of this word in this context.\n\n        The arguments are the same as for `score` and `unmasked_score`.\n\n        \"\"\"\n        return log_base2(self.score(word, context))\n\n    def context_counts(self, context):\n        \"\"\"Helper method for retrieving counts for a given context.\n\n        Assumes context has been checked and oov words in it masked.\n        :type context: tuple(str) or None\n\n        \"\"\"\n        return (\n            self.counts[len(context) + 1][context] if context else self.counts.unigrams\n        )\n\n    def entropy(self, text_ngrams):\n        \"\"\"Calculate cross-entropy of model for given evaluation text.\n\n        :param Iterable(tuple(str)) text_ngrams: A sequence of ngram tuples.\n        :rtype: float\n\n        \"\"\"\n        return -1 * _mean(\n            [self.logscore(ngram[-1], ngram[:-1]) for ngram in text_ngrams]\n        )\n\n    def perplexity(self, text_ngrams):\n        \"\"\"Calculates the perplexity of the given text.\n\n        This is simply 2 ** cross-entropy for the text, so the arguments are the same.\n\n        \"\"\"\n        return pow(2.0, self.entropy(text_ngrams))\n\n    def generate(self, num_words=1, text_seed=None, random_seed=None):\n        \"\"\"Generate words from the model.\n\n        :param int num_words: How many words to generate. By default 1.\n        :param text_seed: Generation can be conditioned on preceding context.\n        :param random_seed: A random seed or an instance of `random.Random`. If provided,\n            makes the random sampling part of generation reproducible.\n        :return: One (str) word or a list of words generated from model.\n\n        Examples:\n\n        >>> from nltk.lm import MLE\n        >>> lm = MLE(2)\n        >>> lm.fit([[(\"a\", \"b\"), (\"b\", \"c\")]], vocabulary_text=['a', 'b', 'c'])\n        >>> lm.fit([[(\"a\",), (\"b\",), (\"c\",)]])\n        >>> lm.generate(random_seed=3)\n        'a'\n        >>> lm.generate(text_seed=['a'])\n        'b'\n\n        \"\"\"\n        text_seed = [] if text_seed is None else list(text_seed)\n        random_generator = _random_generator(random_seed)\n        # This is the base recursion case.\n        if num_words == 1:\n            context = (\n                text_seed[-self.order + 1 :]\n                if len(text_seed) >= self.order\n                else text_seed\n            )\n            samples = self.context_counts(self.vocab.lookup(context))\n            while context and not samples:\n                context = context[1:] if len(context) > 1 else []\n                samples = self.context_counts(self.vocab.lookup(context))\n            # Sorting samples achieves two things:\n            # - reproducible randomness when sampling\n            # - turns Mapping into Sequence which `_weighted_choice` expects\n            samples = sorted(samples)\n            return _weighted_choice(\n                samples,\n                tuple(self.score(w, context) for w in samples),\n                random_generator,\n            )\n        # We build up text one word at a time using the preceding context.\n        generated = []\n        for _ in range(num_words):\n            generated.append(\n                self.generate(\n                    num_words=1,\n                    text_seed=text_seed + generated,\n                    random_seed=random_generator,\n                )\n            )\n        return generated\n", "path": "nltk/lm/api.py"}], "after_files": [{"content": "# Natural Language Toolkit: Language Models\n#\n# Copyright (C) 2001-2022 NLTK Project\n# Authors: Ilia Kurenkov <ilia.kurenkov@gmail.com>\n# URL: <https://www.nltk.org/>\n# For license information, see LICENSE.TXT\n\"\"\"Language Model Interface.\"\"\"\n\nimport random\nimport warnings\nfrom abc import ABCMeta, abstractmethod\nfrom bisect import bisect\nfrom itertools import accumulate\n\nfrom nltk.lm.counter import NgramCounter\nfrom nltk.lm.util import log_base2\nfrom nltk.lm.vocabulary import Vocabulary\n\n\nclass Smoothing(metaclass=ABCMeta):\n    \"\"\"Ngram Smoothing Interface\n\n    Implements Chen & Goodman 1995's idea that all smoothing algorithms have\n    certain features in common. This should ideally allow smoothing algorithms to\n    work both with Backoff and Interpolation.\n    \"\"\"\n\n    def __init__(self, vocabulary, counter):\n        \"\"\"\n        :param vocabulary: The Ngram vocabulary object.\n        :type vocabulary: nltk.lm.vocab.Vocabulary\n        :param counter: The counts of the vocabulary items.\n        :type counter: nltk.lm.counter.NgramCounter\n        \"\"\"\n        self.vocab = vocabulary\n        self.counts = counter\n\n    @abstractmethod\n    def unigram_score(self, word):\n        raise NotImplementedError()\n\n    @abstractmethod\n    def alpha_gamma(self, word, context):\n        raise NotImplementedError()\n\n\ndef _mean(items):\n    \"\"\"Return average (aka mean) for sequence of items.\"\"\"\n    return sum(items) / len(items)\n\n\ndef _random_generator(seed_or_generator):\n    if isinstance(seed_or_generator, random.Random):\n        return seed_or_generator\n    return random.Random(seed_or_generator)\n\n\ndef _weighted_choice(population, weights, random_generator=None):\n    \"\"\"Like random.choice, but with weights.\n\n    Heavily inspired by python 3.6 `random.choices`.\n    \"\"\"\n    if not population:\n        raise ValueError(\"Can't choose from empty population\")\n    if len(population) != len(weights):\n        raise ValueError(\"The number of weights does not match the population\")\n    cum_weights = list(accumulate(weights))\n    total = cum_weights[-1]\n    threshold = random_generator.random()\n    return population[bisect(cum_weights, total * threshold)]\n\n\nclass LanguageModel(metaclass=ABCMeta):\n    \"\"\"ABC for Language Models.\n\n    Cannot be directly instantiated itself.\n\n    \"\"\"\n\n    def __init__(self, order, vocabulary=None, counter=None):\n        \"\"\"Creates new LanguageModel.\n\n        :param vocabulary: If provided, this vocabulary will be used instead\n            of creating a new one when training.\n        :type vocabulary: `nltk.lm.Vocabulary` or None\n        :param counter: If provided, use this object to count ngrams.\n        :type counter: `nltk.lm.NgramCounter` or None\n        :param ngrams_fn: If given, defines how sentences in training text are turned to ngram\n            sequences.\n        :type ngrams_fn: function or None\n        :param pad_fn: If given, defines how sentences in training text are padded.\n        :type pad_fn: function or None\n        \"\"\"\n        self.order = order\n        if vocabulary and not isinstance(vocabulary, Vocabulary):\n            warnings.warn(\n                f\"The `vocabulary` argument passed to {self.__class__.__name__!r} \"\n                \"must be an instance of `nltk.lm.Vocabulary`.\",\n                stacklevel=3,\n            )\n        self.vocab = Vocabulary() if vocabulary is None else vocabulary\n        self.counts = NgramCounter() if counter is None else counter\n\n    def fit(self, text, vocabulary_text=None):\n        \"\"\"Trains the model on a text.\n\n        :param text: Training text as a sequence of sentences.\n\n        \"\"\"\n        if not self.vocab:\n            if vocabulary_text is None:\n                raise ValueError(\n                    \"Cannot fit without a vocabulary or text to create it from.\"\n                )\n            self.vocab.update(vocabulary_text)\n        self.counts.update(self.vocab.lookup(sent) for sent in text)\n\n    def score(self, word, context=None):\n        \"\"\"Masks out of vocab (OOV) words and computes their model score.\n\n        For model-specific logic of calculating scores, see the `unmasked_score`\n        method.\n        \"\"\"\n        return self.unmasked_score(\n            self.vocab.lookup(word), self.vocab.lookup(context) if context else None\n        )\n\n    @abstractmethod\n    def unmasked_score(self, word, context=None):\n        \"\"\"Score a word given some optional context.\n\n        Concrete models are expected to provide an implementation.\n        Note that this method does not mask its arguments with the OOV label.\n        Use the `score` method for that.\n\n        :param str word: Word for which we want the score\n        :param tuple(str) context: Context the word is in.\n            If `None`, compute unigram score.\n        :param context: tuple(str) or None\n        :rtype: float\n        \"\"\"\n        raise NotImplementedError()\n\n    def logscore(self, word, context=None):\n        \"\"\"Evaluate the log score of this word in this context.\n\n        The arguments are the same as for `score` and `unmasked_score`.\n\n        \"\"\"\n        return log_base2(self.score(word, context))\n\n    def context_counts(self, context):\n        \"\"\"Helper method for retrieving counts for a given context.\n\n        Assumes context has been checked and oov words in it masked.\n        :type context: tuple(str) or None\n\n        \"\"\"\n        return (\n            self.counts[len(context) + 1][context] if context else self.counts.unigrams\n        )\n\n    def entropy(self, text_ngrams):\n        \"\"\"Calculate cross-entropy of model for given evaluation text.\n\n        :param Iterable(tuple(str)) text_ngrams: A sequence of ngram tuples.\n        :rtype: float\n\n        \"\"\"\n        return -1 * _mean(\n            [self.logscore(ngram[-1], ngram[:-1]) for ngram in text_ngrams]\n        )\n\n    def perplexity(self, text_ngrams):\n        \"\"\"Calculates the perplexity of the given text.\n\n        This is simply 2 ** cross-entropy for the text, so the arguments are the same.\n\n        \"\"\"\n        return pow(2.0, self.entropy(text_ngrams))\n\n    def generate(self, num_words=1, text_seed=None, random_seed=None):\n        \"\"\"Generate words from the model.\n\n        :param int num_words: How many words to generate. By default 1.\n        :param text_seed: Generation can be conditioned on preceding context.\n        :param random_seed: A random seed or an instance of `random.Random`. If provided,\n            makes the random sampling part of generation reproducible.\n        :return: One (str) word or a list of words generated from model.\n\n        Examples:\n\n        >>> from nltk.lm import MLE\n        >>> lm = MLE(2)\n        >>> lm.fit([[(\"a\", \"b\"), (\"b\", \"c\")]], vocabulary_text=['a', 'b', 'c'])\n        >>> lm.fit([[(\"a\",), (\"b\",), (\"c\",)]])\n        >>> lm.generate(random_seed=3)\n        'a'\n        >>> lm.generate(text_seed=['a'])\n        'b'\n\n        \"\"\"\n        text_seed = [] if text_seed is None else list(text_seed)\n        random_generator = _random_generator(random_seed)\n        # This is the base recursion case.\n        if num_words == 1:\n            context = (\n                text_seed[-self.order + 1 :]\n                if len(text_seed) >= self.order\n                else text_seed\n            )\n            samples = self.context_counts(self.vocab.lookup(context))\n            while context and not samples:\n                context = context[1:] if len(context) > 1 else []\n                samples = self.context_counts(self.vocab.lookup(context))\n            # Sorting samples achieves two things:\n            # - reproducible randomness when sampling\n            # - turns Mapping into Sequence which `_weighted_choice` expects\n            samples = sorted(samples)\n            return _weighted_choice(\n                samples,\n                tuple(self.score(w, context) for w in samples),\n                random_generator,\n            )\n        # We build up text one word at a time using the preceding context.\n        generated = []\n        for _ in range(num_words):\n            generated.append(\n                self.generate(\n                    num_words=1,\n                    text_seed=text_seed + generated,\n                    random_seed=random_generator,\n                )\n            )\n        return generated\n", "path": "nltk/lm/api.py"}]}