We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
BllipParser: AttributeError: 'str' object has no attribute 'decode' on Python 3.5
When calling next(BllipParser.parse()) on Python 3.5, I receive the following error:

File "/home/andrew/.local/lib/python3.5/site-packages/nltk/parse/bllip.py", line 169, in parse
    _ensure_ascii(sentence)
  File "/home/andrew/.local/lib/python3.5/site-packages/nltk/parse/bllip.py", line 101, in _ensure_ascii
    word.decode('ascii')
AttributeError: 'str' object has no attribute 'decode'


Related to https://github.com/nltk/nltk/issues/507

NLTK v3.2.5
Python v3.5.2
OS: Ubuntu (16.04.3) running on WSL (16299.125)

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `nltk/parse/bllip.py`
Content:
```
1 # Natural Language Toolkit: Interface to BLLIP Parser
2 #
3 # Author: David McClosky <dmcc@bigasterisk.com>
4 #
5 # Copyright (C) 2001-2021 NLTK Project
6 # URL: <https://www.nltk.org/>
7 # For license information, see LICENSE.TXT
8 
9 from nltk.parse.api import ParserI
10 from nltk.tree import Tree
11 
12 """
13 Interface for parsing with BLLIP Parser. Requires the Python
14 bllipparser module. BllipParser objects can be constructed with the
15 ``BllipParser.from_unified_model_dir`` class method or manually using the
16 ``BllipParser`` constructor. The former is generally easier if you have
17 a BLLIP Parser unified model directory -- a basic model can be obtained
18 from NLTK's downloader. More unified parsing models can be obtained with
19 BLLIP Parser's ModelFetcher (run ``python -m bllipparser.ModelFetcher``
20 or see docs for ``bllipparser.ModelFetcher.download_and_install_model``).
21 
22 Basic usage::
23 
24     # download and install a basic unified parsing model (Wall Street Journal)
25     # sudo python -m nltk.downloader bllip_wsj_no_aux
26 
27     >>> from nltk.data import find
28     >>> model_dir = find('models/bllip_wsj_no_aux').path
29     >>> bllip = BllipParser.from_unified_model_dir(model_dir)
30 
31     # 1-best parsing
32     >>> sentence1 = 'British left waffles on Falklands .'.split()
33     >>> top_parse = bllip.parse_one(sentence1)
34     >>> print(top_parse)
35     (S1
36       (S
37         (NP (JJ British) (NN left))
38         (VP (VBZ waffles) (PP (IN on) (NP (NNP Falklands))))
39         (. .)))
40 
41     # n-best parsing
42     >>> sentence2 = 'Time flies'.split()
43     >>> all_parses = bllip.parse_all(sentence2)
44     >>> print(len(all_parses))
45     50
46     >>> print(all_parses[0])
47     (S1 (S (NP (NNP Time)) (VP (VBZ flies))))
48 
49     # incorporating external tagging constraints (None means unconstrained tag)
50     >>> constrained1 = bllip.tagged_parse([('Time', 'VB'), ('flies', 'NNS')])
51     >>> print(next(constrained1))
52     (S1 (NP (VB Time) (NNS flies)))
53     >>> constrained2 = bllip.tagged_parse([('Time', 'NN'), ('flies', None)])
54     >>> print(next(constrained2))
55     (S1 (NP (NN Time) (VBZ flies)))
56 
57 References
58 ----------
59 
60 - Charniak, Eugene. "A maximum-entropy-inspired parser." Proceedings of
61   the 1st North American chapter of the Association for Computational
62   Linguistics conference. Association for Computational Linguistics,
63   2000.
64 
65 - Charniak, Eugene, and Mark Johnson. "Coarse-to-fine n-best parsing
66   and MaxEnt discriminative reranking." Proceedings of the 43rd Annual
67   Meeting on Association for Computational Linguistics. Association
68   for Computational Linguistics, 2005.
69 
70 Known issues
71 ------------
72 
73 Note that BLLIP Parser is not currently threadsafe. Since this module
74 uses a SWIG interface, it is potentially unsafe to create multiple
75 ``BllipParser`` objects in the same process. BLLIP Parser currently
76 has issues with non-ASCII text and will raise an error if given any.
77 
78 See https://pypi.python.org/pypi/bllipparser/ for more information
79 on BLLIP Parser's Python interface.
80 """
81 
82 __all__ = ["BllipParser"]
83 
84 # this block allows this module to be imported even if bllipparser isn't
85 # available
86 try:
87     from bllipparser import RerankingParser
88     from bllipparser.RerankingParser import get_unified_model_parameters
89 
90     def _ensure_bllip_import_or_error():
91         pass
92 
93 
94 except ImportError as ie:
95 
96     def _ensure_bllip_import_or_error(ie=ie):
97         raise ImportError("Couldn't import bllipparser module: %s" % ie)
98 
99 
100 def _ensure_ascii(words):
101     try:
102         for i, word in enumerate(words):
103             word.decode("ascii")
104     except UnicodeDecodeError as e:
105         raise ValueError(
106             "Token %d (%r) is non-ASCII. BLLIP Parser "
107             "currently doesn't support non-ASCII inputs." % (i, word)
108         ) from e
109 
110 
111 def _scored_parse_to_nltk_tree(scored_parse):
112     return Tree.fromstring(str(scored_parse.ptb_parse))
113 
114 
115 class BllipParser(ParserI):
116     """
117     Interface for parsing with BLLIP Parser. BllipParser objects can be
118     constructed with the ``BllipParser.from_unified_model_dir`` class
119     method or manually using the ``BllipParser`` constructor.
120     """
121 
122     def __init__(
123         self,
124         parser_model=None,
125         reranker_features=None,
126         reranker_weights=None,
127         parser_options=None,
128         reranker_options=None,
129     ):
130         """
131         Load a BLLIP Parser model from scratch. You'll typically want to
132         use the ``from_unified_model_dir()`` class method to construct
133         this object.
134 
135         :param parser_model: Path to parser model directory
136         :type parser_model: str
137 
138         :param reranker_features: Path the reranker model's features file
139         :type reranker_features: str
140 
141         :param reranker_weights: Path the reranker model's weights file
142         :type reranker_weights: str
143 
144         :param parser_options: optional dictionary of parser options, see
145             ``bllipparser.RerankingParser.RerankingParser.load_parser_options()``
146             for more information.
147         :type parser_options: dict(str)
148 
149         :param reranker_options: optional
150             dictionary of reranker options, see
151             ``bllipparser.RerankingParser.RerankingParser.load_reranker_model()``
152             for more information.
153         :type reranker_options: dict(str)
154         """
155         _ensure_bllip_import_or_error()
156 
157         parser_options = parser_options or {}
158         reranker_options = reranker_options or {}
159 
160         self.rrp = RerankingParser()
161         self.rrp.load_parser_model(parser_model, **parser_options)
162         if reranker_features and reranker_weights:
163             self.rrp.load_reranker_model(
164                 features_filename=reranker_features,
165                 weights_filename=reranker_weights,
166                 **reranker_options
167             )
168 
169     def parse(self, sentence):
170         """
171         Use BLLIP Parser to parse a sentence. Takes a sentence as a list
172         of words; it will be automatically tagged with this BLLIP Parser
173         instance's tagger.
174 
175         :return: An iterator that generates parse trees for the sentence
176             from most likely to least likely.
177 
178         :param sentence: The sentence to be parsed
179         :type sentence: list(str)
180         :rtype: iter(Tree)
181         """
182         _ensure_ascii(sentence)
183         nbest_list = self.rrp.parse(sentence)
184         for scored_parse in nbest_list:
185             yield _scored_parse_to_nltk_tree(scored_parse)
186 
187     def tagged_parse(self, word_and_tag_pairs):
188         """
189         Use BLLIP to parse a sentence. Takes a sentence as a list of
190         (word, tag) tuples; the sentence must have already been tokenized
191         and tagged. BLLIP will attempt to use the tags provided but may
192         use others if it can't come up with a complete parse subject
193         to those constraints. You may also specify a tag as ``None``
194         to leave a token's tag unconstrained.
195 
196         :return: An iterator that generates parse trees for the sentence
197             from most likely to least likely.
198 
199         :param sentence: Input sentence to parse as (word, tag) pairs
200         :type sentence: list(tuple(str, str))
201         :rtype: iter(Tree)
202         """
203         words = []
204         tag_map = {}
205         for i, (word, tag) in enumerate(word_and_tag_pairs):
206             words.append(word)
207             if tag is not None:
208                 tag_map[i] = tag
209 
210         _ensure_ascii(words)
211         nbest_list = self.rrp.parse_tagged(words, tag_map)
212         for scored_parse in nbest_list:
213             yield _scored_parse_to_nltk_tree(scored_parse)
214 
215     @classmethod
216     def from_unified_model_dir(
217         cls, model_dir, parser_options=None, reranker_options=None
218     ):
219         """
220         Create a ``BllipParser`` object from a unified parsing model
221         directory. Unified parsing model directories are a standardized
222         way of storing BLLIP parser and reranker models together on disk.
223         See ``bllipparser.RerankingParser.get_unified_model_parameters()``
224         for more information about unified model directories.
225 
226         :return: A ``BllipParser`` object using the parser and reranker
227             models in the model directory.
228 
229         :param model_dir: Path to the unified model directory.
230         :type model_dir: str
231         :param parser_options: optional dictionary of parser options, see
232             ``bllipparser.RerankingParser.RerankingParser.load_parser_options()``
233             for more information.
234         :type parser_options: dict(str)
235         :param reranker_options: optional dictionary of reranker options, see
236             ``bllipparser.RerankingParser.RerankingParser.load_reranker_model()``
237             for more information.
238         :type reranker_options: dict(str)
239         :rtype: BllipParser
240         """
241         (
242             parser_model_dir,
243             reranker_features_filename,
244             reranker_weights_filename,
245         ) = get_unified_model_parameters(model_dir)
246         return cls(
247             parser_model_dir,
248             reranker_features_filename,
249             reranker_weights_filename,
250             parser_options,
251             reranker_options,
252         )
253 
254 
255 def demo():
256     """This assumes the Python module bllipparser is installed."""
257 
258     # download and install a basic unified parsing model (Wall Street Journal)
259     # sudo python -m nltk.downloader bllip_wsj_no_aux
260 
261     from nltk.data import find
262 
263     model_dir = find("models/bllip_wsj_no_aux").path
264 
265     print("Loading BLLIP Parsing models...")
266     # the easiest way to get started is to use a unified model
267     bllip = BllipParser.from_unified_model_dir(model_dir)
268     print("Done.")
269 
270     sentence1 = "British left waffles on Falklands .".split()
271     sentence2 = "I saw the man with the telescope .".split()
272     # this sentence is known to fail under the WSJ parsing model
273     fail1 = "# ! ? : -".split()
274     for sentence in (sentence1, sentence2, fail1):
275         print("Sentence: %r" % " ".join(sentence))
276         try:
277             tree = next(bllip.parse(sentence))
278             print(tree)
279         except StopIteration:
280             print("(parse failed)")
281 
282     # n-best parsing demo
283     for i, parse in enumerate(bllip.parse(sentence1)):
284         print("parse %d:\n%s" % (i, parse))
285 
286     # using external POS tag constraints
287     print(
288         "forcing 'tree' to be 'NN':",
289         next(bllip.tagged_parse([("A", None), ("tree", "NN")])),
290     )
291     print(
292         "forcing 'A' to be 'DT' and 'tree' to be 'NNP':",
293         next(bllip.tagged_parse([("A", "DT"), ("tree", "NNP")])),
294     )
295     # constraints don't have to make sense... (though on more complicated
296     # sentences, they may cause the parse to fail)
297     print(
298         "forcing 'A' to be 'NNP':",
299         next(bllip.tagged_parse([("A", "NNP"), ("tree", None)])),
300     )
301
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/nltk/parse/bllip.py b/nltk/parse/bllip.py
--- a/nltk/parse/bllip.py
+++ b/nltk/parse/bllip.py
@@ -100,11 +100,11 @@
 def _ensure_ascii(words):
     try:
         for i, word in enumerate(words):
-            word.decode("ascii")
-    except UnicodeDecodeError as e:
+            word.encode("ascii")
+    except UnicodeEncodeError as e:
         raise ValueError(
-            "Token %d (%r) is non-ASCII. BLLIP Parser "
-            "currently doesn't support non-ASCII inputs." % (i, word)
+            f"Token {i} ({word!r}) is non-ASCII. BLLIP Parser "
+            "currently doesn't support non-ASCII inputs."
         ) from e
 
 
@@ -163,7 +163,7 @@
             self.rrp.load_reranker_model(
                 features_filename=reranker_features,
                 weights_filename=reranker_weights,
-                **reranker_options
+                **reranker_options,
             )
 
     def parse(self, sentence):

{"golden_diff": "diff --git a/nltk/parse/bllip.py b/nltk/parse/bllip.py\n--- a/nltk/parse/bllip.py\n+++ b/nltk/parse/bllip.py\n@@ -100,11 +100,11 @@\n def _ensure_ascii(words):\n     try:\n         for i, word in enumerate(words):\n-            word.decode(\"ascii\")\n-    except UnicodeDecodeError as e:\n+            word.encode(\"ascii\")\n+    except UnicodeEncodeError as e:\n         raise ValueError(\n-            \"Token %d (%r) is non-ASCII. BLLIP Parser \"\n-            \"currently doesn't support non-ASCII inputs.\" % (i, word)\n+            f\"Token {i} ({word!r}) is non-ASCII. BLLIP Parser \"\n+            \"currently doesn't support non-ASCII inputs.\"\n         ) from e\n \n \n@@ -163,7 +163,7 @@\n             self.rrp.load_reranker_model(\n                 features_filename=reranker_features,\n                 weights_filename=reranker_weights,\n-                **reranker_options\n+                **reranker_options,\n             )\n \n     def parse(self, sentence):\n", "issue": "BllipParser: AttributeError: 'str' object has no attribute 'decode' on Python 3.5\nWhen calling next(BllipParser.parse()) on Python 3.5, I receive the following error:\r\n\r\nFile \"/home/andrew/.local/lib/python3.5/site-packages/nltk/parse/bllip.py\", line 169, in parse\r\n    _ensure_ascii(sentence)\r\n  File \"/home/andrew/.local/lib/python3.5/site-packages/nltk/parse/bllip.py\", line 101, in _ensure_ascii\r\n    word.decode('ascii')\r\nAttributeError: 'str' object has no attribute 'decode'\r\n\r\n\r\nRelated to https://github.com/nltk/nltk/issues/507\r\n\r\nNLTK v3.2.5\r\nPython v3.5.2\r\nOS: Ubuntu (16.04.3) running on WSL (16299.125)\n", "before_files": [{"content": "# Natural Language Toolkit: Interface to BLLIP Parser\n#\n# Author: David McClosky <dmcc@bigasterisk.com>\n#\n# Copyright (C) 2001-2021 NLTK Project\n# URL: <https://www.nltk.org/>\n# For license information, see LICENSE.TXT\n\nfrom nltk.parse.api import ParserI\nfrom nltk.tree import Tree\n\n\"\"\"\nInterface for parsing with BLLIP Parser. Requires the Python\nbllipparser module. BllipParser objects can be constructed with the\n``BllipParser.from_unified_model_dir`` class method or manually using the\n``BllipParser`` constructor. The former is generally easier if you have\na BLLIP Parser unified model directory -- a basic model can be obtained\nfrom NLTK's downloader. More unified parsing models can be obtained with\nBLLIP Parser's ModelFetcher (run ``python -m bllipparser.ModelFetcher``\nor see docs for ``bllipparser.ModelFetcher.download_and_install_model``).\n\nBasic usage::\n\n    # download and install a basic unified parsing model (Wall Street Journal)\n    # sudo python -m nltk.downloader bllip_wsj_no_aux\n\n    >>> from nltk.data import find\n    >>> model_dir = find('models/bllip_wsj_no_aux').path\n    >>> bllip = BllipParser.from_unified_model_dir(model_dir)\n\n    # 1-best parsing\n    >>> sentence1 = 'British left waffles on Falklands .'.split()\n    >>> top_parse = bllip.parse_one(sentence1)\n    >>> print(top_parse)\n    (S1\n      (S\n        (NP (JJ British) (NN left))\n        (VP (VBZ waffles) (PP (IN on) (NP (NNP Falklands))))\n        (. .)))\n\n    # n-best parsing\n    >>> sentence2 = 'Time flies'.split()\n    >>> all_parses = bllip.parse_all(sentence2)\n    >>> print(len(all_parses))\n    50\n    >>> print(all_parses[0])\n    (S1 (S (NP (NNP Time)) (VP (VBZ flies))))\n\n    # incorporating external tagging constraints (None means unconstrained tag)\n    >>> constrained1 = bllip.tagged_parse([('Time', 'VB'), ('flies', 'NNS')])\n    >>> print(next(constrained1))\n    (S1 (NP (VB Time) (NNS flies)))\n    >>> constrained2 = bllip.tagged_parse([('Time', 'NN'), ('flies', None)])\n    >>> print(next(constrained2))\n    (S1 (NP (NN Time) (VBZ flies)))\n\nReferences\n----------\n\n- Charniak, Eugene. \"A maximum-entropy-inspired parser.\" Proceedings of\n  the 1st North American chapter of the Association for Computational\n  Linguistics conference. Association for Computational Linguistics,\n  2000.\n\n- Charniak, Eugene, and Mark Johnson. \"Coarse-to-fine n-best parsing\n  and MaxEnt discriminative reranking.\" Proceedings of the 43rd Annual\n  Meeting on Association for Computational Linguistics. Association\n  for Computational Linguistics, 2005.\n\nKnown issues\n------------\n\nNote that BLLIP Parser is not currently threadsafe. Since this module\nuses a SWIG interface, it is potentially unsafe to create multiple\n``BllipParser`` objects in the same process. BLLIP Parser currently\nhas issues with non-ASCII text and will raise an error if given any.\n\nSee https://pypi.python.org/pypi/bllipparser/ for more information\non BLLIP Parser's Python interface.\n\"\"\"\n\n__all__ = [\"BllipParser\"]\n\n# this block allows this module to be imported even if bllipparser isn't\n# available\ntry:\n    from bllipparser import RerankingParser\n    from bllipparser.RerankingParser import get_unified_model_parameters\n\n    def _ensure_bllip_import_or_error():\n        pass\n\n\nexcept ImportError as ie:\n\n    def _ensure_bllip_import_or_error(ie=ie):\n        raise ImportError(\"Couldn't import bllipparser module: %s\" % ie)\n\n\ndef _ensure_ascii(words):\n    try:\n        for i, word in enumerate(words):\n            word.decode(\"ascii\")\n    except UnicodeDecodeError as e:\n        raise ValueError(\n            \"Token %d (%r) is non-ASCII. BLLIP Parser \"\n            \"currently doesn't support non-ASCII inputs.\" % (i, word)\n        ) from e\n\n\ndef _scored_parse_to_nltk_tree(scored_parse):\n    return Tree.fromstring(str(scored_parse.ptb_parse))\n\n\nclass BllipParser(ParserI):\n    \"\"\"\n    Interface for parsing with BLLIP Parser. BllipParser objects can be\n    constructed with the ``BllipParser.from_unified_model_dir`` class\n    method or manually using the ``BllipParser`` constructor.\n    \"\"\"\n\n    def __init__(\n        self,\n        parser_model=None,\n        reranker_features=None,\n        reranker_weights=None,\n        parser_options=None,\n        reranker_options=None,\n    ):\n        \"\"\"\n        Load a BLLIP Parser model from scratch. You'll typically want to\n        use the ``from_unified_model_dir()`` class method to construct\n        this object.\n\n        :param parser_model: Path to parser model directory\n        :type parser_model: str\n\n        :param reranker_features: Path the reranker model's features file\n        :type reranker_features: str\n\n        :param reranker_weights: Path the reranker model's weights file\n        :type reranker_weights: str\n\n        :param parser_options: optional dictionary of parser options, see\n            ``bllipparser.RerankingParser.RerankingParser.load_parser_options()``\n            for more information.\n        :type parser_options: dict(str)\n\n        :param reranker_options: optional\n            dictionary of reranker options, see\n            ``bllipparser.RerankingParser.RerankingParser.load_reranker_model()``\n            for more information.\n        :type reranker_options: dict(str)\n        \"\"\"\n        _ensure_bllip_import_or_error()\n\n        parser_options = parser_options or {}\n        reranker_options = reranker_options or {}\n\n        self.rrp = RerankingParser()\n        self.rrp.load_parser_model(parser_model, **parser_options)\n        if reranker_features and reranker_weights:\n            self.rrp.load_reranker_model(\n                features_filename=reranker_features,\n                weights_filename=reranker_weights,\n                **reranker_options\n            )\n\n    def parse(self, sentence):\n        \"\"\"\n        Use BLLIP Parser to parse a sentence. Takes a sentence as a list\n        of words; it will be automatically tagged with this BLLIP Parser\n        instance's tagger.\n\n        :return: An iterator that generates parse trees for the sentence\n            from most likely to least likely.\n\n        :param sentence: The sentence to be parsed\n        :type sentence: list(str)\n        :rtype: iter(Tree)\n        \"\"\"\n        _ensure_ascii(sentence)\n        nbest_list = self.rrp.parse(sentence)\n        for scored_parse in nbest_list:\n            yield _scored_parse_to_nltk_tree(scored_parse)\n\n    def tagged_parse(self, word_and_tag_pairs):\n        \"\"\"\n        Use BLLIP to parse a sentence. Takes a sentence as a list of\n        (word, tag) tuples; the sentence must have already been tokenized\n        and tagged. BLLIP will attempt to use the tags provided but may\n        use others if it can't come up with a complete parse subject\n        to those constraints. You may also specify a tag as ``None``\n        to leave a token's tag unconstrained.\n\n        :return: An iterator that generates parse trees for the sentence\n            from most likely to least likely.\n\n        :param sentence: Input sentence to parse as (word, tag) pairs\n        :type sentence: list(tuple(str, str))\n        :rtype: iter(Tree)\n        \"\"\"\n        words = []\n        tag_map = {}\n        for i, (word, tag) in enumerate(word_and_tag_pairs):\n            words.append(word)\n            if tag is not None:\n                tag_map[i] = tag\n\n        _ensure_ascii(words)\n        nbest_list = self.rrp.parse_tagged(words, tag_map)\n        for scored_parse in nbest_list:\n            yield _scored_parse_to_nltk_tree(scored_parse)\n\n    @classmethod\n    def from_unified_model_dir(\n        cls, model_dir, parser_options=None, reranker_options=None\n    ):\n        \"\"\"\n        Create a ``BllipParser`` object from a unified parsing model\n        directory. Unified parsing model directories are a standardized\n        way of storing BLLIP parser and reranker models together on disk.\n        See ``bllipparser.RerankingParser.get_unified_model_parameters()``\n        for more information about unified model directories.\n\n        :return: A ``BllipParser`` object using the parser and reranker\n            models in the model directory.\n\n        :param model_dir: Path to the unified model directory.\n        :type model_dir: str\n        :param parser_options: optional dictionary of parser options, see\n            ``bllipparser.RerankingParser.RerankingParser.load_parser_options()``\n            for more information.\n        :type parser_options: dict(str)\n        :param reranker_options: optional dictionary of reranker options, see\n            ``bllipparser.RerankingParser.RerankingParser.load_reranker_model()``\n            for more information.\n        :type reranker_options: dict(str)\n        :rtype: BllipParser\n        \"\"\"\n        (\n            parser_model_dir,\n            reranker_features_filename,\n            reranker_weights_filename,\n        ) = get_unified_model_parameters(model_dir)\n        return cls(\n            parser_model_dir,\n            reranker_features_filename,\n            reranker_weights_filename,\n            parser_options,\n            reranker_options,\n        )\n\n\ndef demo():\n    \"\"\"This assumes the Python module bllipparser is installed.\"\"\"\n\n    # download and install a basic unified parsing model (Wall Street Journal)\n    # sudo python -m nltk.downloader bllip_wsj_no_aux\n\n    from nltk.data import find\n\n    model_dir = find(\"models/bllip_wsj_no_aux\").path\n\n    print(\"Loading BLLIP Parsing models...\")\n    # the easiest way to get started is to use a unified model\n    bllip = BllipParser.from_unified_model_dir(model_dir)\n    print(\"Done.\")\n\n    sentence1 = \"British left waffles on Falklands .\".split()\n    sentence2 = \"I saw the man with the telescope .\".split()\n    # this sentence is known to fail under the WSJ parsing model\n    fail1 = \"# ! ? : -\".split()\n    for sentence in (sentence1, sentence2, fail1):\n        print(\"Sentence: %r\" % \" \".join(sentence))\n        try:\n            tree = next(bllip.parse(sentence))\n            print(tree)\n        except StopIteration:\n            print(\"(parse failed)\")\n\n    # n-best parsing demo\n    for i, parse in enumerate(bllip.parse(sentence1)):\n        print(\"parse %d:\\n%s\" % (i, parse))\n\n    # using external POS tag constraints\n    print(\n        \"forcing 'tree' to be 'NN':\",\n        next(bllip.tagged_parse([(\"A\", None), (\"tree\", \"NN\")])),\n    )\n    print(\n        \"forcing 'A' to be 'DT' and 'tree' to be 'NNP':\",\n        next(bllip.tagged_parse([(\"A\", \"DT\"), (\"tree\", \"NNP\")])),\n    )\n    # constraints don't have to make sense... (though on more complicated\n    # sentences, they may cause the parse to fail)\n    print(\n        \"forcing 'A' to be 'NNP':\",\n        next(bllip.tagged_parse([(\"A\", \"NNP\"), (\"tree\", None)])),\n    )\n", "path": "nltk/parse/bllip.py"}], "after_files": [{"content": "# Natural Language Toolkit: Interface to BLLIP Parser\n#\n# Author: David McClosky <dmcc@bigasterisk.com>\n#\n# Copyright (C) 2001-2021 NLTK Project\n# URL: <https://www.nltk.org/>\n# For license information, see LICENSE.TXT\n\nfrom nltk.parse.api import ParserI\nfrom nltk.tree import Tree\n\n\"\"\"\nInterface for parsing with BLLIP Parser. Requires the Python\nbllipparser module. BllipParser objects can be constructed with the\n``BllipParser.from_unified_model_dir`` class method or manually using the\n``BllipParser`` constructor. The former is generally easier if you have\na BLLIP Parser unified model directory -- a basic model can be obtained\nfrom NLTK's downloader. More unified parsing models can be obtained with\nBLLIP Parser's ModelFetcher (run ``python -m bllipparser.ModelFetcher``\nor see docs for ``bllipparser.ModelFetcher.download_and_install_model``).\n\nBasic usage::\n\n    # download and install a basic unified parsing model (Wall Street Journal)\n    # sudo python -m nltk.downloader bllip_wsj_no_aux\n\n    >>> from nltk.data import find\n    >>> model_dir = find('models/bllip_wsj_no_aux').path\n    >>> bllip = BllipParser.from_unified_model_dir(model_dir)\n\n    # 1-best parsing\n    >>> sentence1 = 'British left waffles on Falklands .'.split()\n    >>> top_parse = bllip.parse_one(sentence1)\n    >>> print(top_parse)\n    (S1\n      (S\n        (NP (JJ British) (NN left))\n        (VP (VBZ waffles) (PP (IN on) (NP (NNP Falklands))))\n        (. .)))\n\n    # n-best parsing\n    >>> sentence2 = 'Time flies'.split()\n    >>> all_parses = bllip.parse_all(sentence2)\n    >>> print(len(all_parses))\n    50\n    >>> print(all_parses[0])\n    (S1 (S (NP (NNP Time)) (VP (VBZ flies))))\n\n    # incorporating external tagging constraints (None means unconstrained tag)\n    >>> constrained1 = bllip.tagged_parse([('Time', 'VB'), ('flies', 'NNS')])\n    >>> print(next(constrained1))\n    (S1 (NP (VB Time) (NNS flies)))\n    >>> constrained2 = bllip.tagged_parse([('Time', 'NN'), ('flies', None)])\n    >>> print(next(constrained2))\n    (S1 (NP (NN Time) (VBZ flies)))\n\nReferences\n----------\n\n- Charniak, Eugene. \"A maximum-entropy-inspired parser.\" Proceedings of\n  the 1st North American chapter of the Association for Computational\n  Linguistics conference. Association for Computational Linguistics,\n  2000.\n\n- Charniak, Eugene, and Mark Johnson. \"Coarse-to-fine n-best parsing\n  and MaxEnt discriminative reranking.\" Proceedings of the 43rd Annual\n  Meeting on Association for Computational Linguistics. Association\n  for Computational Linguistics, 2005.\n\nKnown issues\n------------\n\nNote that BLLIP Parser is not currently threadsafe. Since this module\nuses a SWIG interface, it is potentially unsafe to create multiple\n``BllipParser`` objects in the same process. BLLIP Parser currently\nhas issues with non-ASCII text and will raise an error if given any.\n\nSee https://pypi.python.org/pypi/bllipparser/ for more information\non BLLIP Parser's Python interface.\n\"\"\"\n\n__all__ = [\"BllipParser\"]\n\n# this block allows this module to be imported even if bllipparser isn't\n# available\ntry:\n    from bllipparser import RerankingParser\n    from bllipparser.RerankingParser import get_unified_model_parameters\n\n    def _ensure_bllip_import_or_error():\n        pass\n\n\nexcept ImportError as ie:\n\n    def _ensure_bllip_import_or_error(ie=ie):\n        raise ImportError(\"Couldn't import bllipparser module: %s\" % ie)\n\n\ndef _ensure_ascii(words):\n    try:\n        for i, word in enumerate(words):\n            word.encode(\"ascii\")\n    except UnicodeEncodeError as e:\n        raise ValueError(\n            f\"Token {i} ({word!r}) is non-ASCII. BLLIP Parser \"\n            \"currently doesn't support non-ASCII inputs.\"\n        ) from e\n\n\ndef _scored_parse_to_nltk_tree(scored_parse):\n    return Tree.fromstring(str(scored_parse.ptb_parse))\n\n\nclass BllipParser(ParserI):\n    \"\"\"\n    Interface for parsing with BLLIP Parser. BllipParser objects can be\n    constructed with the ``BllipParser.from_unified_model_dir`` class\n    method or manually using the ``BllipParser`` constructor.\n    \"\"\"\n\n    def __init__(\n        self,\n        parser_model=None,\n        reranker_features=None,\n        reranker_weights=None,\n        parser_options=None,\n        reranker_options=None,\n    ):\n        \"\"\"\n        Load a BLLIP Parser model from scratch. You'll typically want to\n        use the ``from_unified_model_dir()`` class method to construct\n        this object.\n\n        :param parser_model: Path to parser model directory\n        :type parser_model: str\n\n        :param reranker_features: Path the reranker model's features file\n        :type reranker_features: str\n\n        :param reranker_weights: Path the reranker model's weights file\n        :type reranker_weights: str\n\n        :param parser_options: optional dictionary of parser options, see\n            ``bllipparser.RerankingParser.RerankingParser.load_parser_options()``\n            for more information.\n        :type parser_options: dict(str)\n\n        :param reranker_options: optional\n            dictionary of reranker options, see\n            ``bllipparser.RerankingParser.RerankingParser.load_reranker_model()``\n            for more information.\n        :type reranker_options: dict(str)\n        \"\"\"\n        _ensure_bllip_import_or_error()\n\n        parser_options = parser_options or {}\n        reranker_options = reranker_options or {}\n\n        self.rrp = RerankingParser()\n        self.rrp.load_parser_model(parser_model, **parser_options)\n        if reranker_features and reranker_weights:\n            self.rrp.load_reranker_model(\n                features_filename=reranker_features,\n                weights_filename=reranker_weights,\n                **reranker_options,\n            )\n\n    def parse(self, sentence):\n        \"\"\"\n        Use BLLIP Parser to parse a sentence. Takes a sentence as a list\n        of words; it will be automatically tagged with this BLLIP Parser\n        instance's tagger.\n\n        :return: An iterator that generates parse trees for the sentence\n            from most likely to least likely.\n\n        :param sentence: The sentence to be parsed\n        :type sentence: list(str)\n        :rtype: iter(Tree)\n        \"\"\"\n        _ensure_ascii(sentence)\n        nbest_list = self.rrp.parse(sentence)\n        for scored_parse in nbest_list:\n            yield _scored_parse_to_nltk_tree(scored_parse)\n\n    def tagged_parse(self, word_and_tag_pairs):\n        \"\"\"\n        Use BLLIP to parse a sentence. Takes a sentence as a list of\n        (word, tag) tuples; the sentence must have already been tokenized\n        and tagged. BLLIP will attempt to use the tags provided but may\n        use others if it can't come up with a complete parse subject\n        to those constraints. You may also specify a tag as ``None``\n        to leave a token's tag unconstrained.\n\n        :return: An iterator that generates parse trees for the sentence\n            from most likely to least likely.\n\n        :param sentence: Input sentence to parse as (word, tag) pairs\n        :type sentence: list(tuple(str, str))\n        :rtype: iter(Tree)\n        \"\"\"\n        words = []\n        tag_map = {}\n        for i, (word, tag) in enumerate(word_and_tag_pairs):\n            words.append(word)\n            if tag is not None:\n                tag_map[i] = tag\n\n        _ensure_ascii(words)\n        nbest_list = self.rrp.parse_tagged(words, tag_map)\n        for scored_parse in nbest_list:\n            yield _scored_parse_to_nltk_tree(scored_parse)\n\n    @classmethod\n    def from_unified_model_dir(\n        cls, model_dir, parser_options=None, reranker_options=None\n    ):\n        \"\"\"\n        Create a ``BllipParser`` object from a unified parsing model\n        directory. Unified parsing model directories are a standardized\n        way of storing BLLIP parser and reranker models together on disk.\n        See ``bllipparser.RerankingParser.get_unified_model_parameters()``\n        for more information about unified model directories.\n\n        :return: A ``BllipParser`` object using the parser and reranker\n            models in the model directory.\n\n        :param model_dir: Path to the unified model directory.\n        :type model_dir: str\n        :param parser_options: optional dictionary of parser options, see\n            ``bllipparser.RerankingParser.RerankingParser.load_parser_options()``\n            for more information.\n        :type parser_options: dict(str)\n        :param reranker_options: optional dictionary of reranker options, see\n            ``bllipparser.RerankingParser.RerankingParser.load_reranker_model()``\n            for more information.\n        :type reranker_options: dict(str)\n        :rtype: BllipParser\n        \"\"\"\n        (\n            parser_model_dir,\n            reranker_features_filename,\n            reranker_weights_filename,\n        ) = get_unified_model_parameters(model_dir)\n        return cls(\n            parser_model_dir,\n            reranker_features_filename,\n            reranker_weights_filename,\n            parser_options,\n            reranker_options,\n        )\n\n\ndef demo():\n    \"\"\"This assumes the Python module bllipparser is installed.\"\"\"\n\n    # download and install a basic unified parsing model (Wall Street Journal)\n    # sudo python -m nltk.downloader bllip_wsj_no_aux\n\n    from nltk.data import find\n\n    model_dir = find(\"models/bllip_wsj_no_aux\").path\n\n    print(\"Loading BLLIP Parsing models...\")\n    # the easiest way to get started is to use a unified model\n    bllip = BllipParser.from_unified_model_dir(model_dir)\n    print(\"Done.\")\n\n    sentence1 = \"British left waffles on Falklands .\".split()\n    sentence2 = \"I saw the man with the telescope .\".split()\n    # this sentence is known to fail under the WSJ parsing model\n    fail1 = \"# ! ? : -\".split()\n    for sentence in (sentence1, sentence2, fail1):\n        print(\"Sentence: %r\" % \" \".join(sentence))\n        try:\n            tree = next(bllip.parse(sentence))\n            print(tree)\n        except StopIteration:\n            print(\"(parse failed)\")\n\n    # n-best parsing demo\n    for i, parse in enumerate(bllip.parse(sentence1)):\n        print(\"parse %d:\\n%s\" % (i, parse))\n\n    # using external POS tag constraints\n    print(\n        \"forcing 'tree' to be 'NN':\",\n        next(bllip.tagged_parse([(\"A\", None), (\"tree\", \"NN\")])),\n    )\n    print(\n        \"forcing 'A' to be 'DT' and 'tree' to be 'NNP':\",\n        next(bllip.tagged_parse([(\"A\", \"DT\"), (\"tree\", \"NNP\")])),\n    )\n    # constraints don't have to make sense... (though on more complicated\n    # sentences, they may cause the parse to fail)\n    print(\n        \"forcing 'A' to be 'NNP':\",\n        next(bllip.tagged_parse([(\"A\", \"NNP\"), (\"tree\", None)])),\n    )\n", "path": "nltk/parse/bllip.py"}]}