We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
`#fmt:skip` causes NTERNAL ERROR: Black produced different code on the second pass of the formatter
**Describe the bug**

A comment `#fmt:skip` with no internal space causes `INTERNAL ERROR: Black produced different code on the second pass of the formatter`

Also the same code, if run with `--fast` creates incorrectly formatted code with multiple copies of the comment on the next line.

**To Reproduce**

For example, take this code:

```python
import re

line = "some dummy data"
log_search = re.search('CLOCK:\s*(\[.*\])--(\[.*\])\s*=>\s*(\d*:\d*)', line)  #fmt:skip
```

And run it with these arguments:

```sh
$ black fmt_skip.py .
```

The resulting error is:

error: cannot format fmt_skip.py: INTERNAL ERROR: Black produced different code on the second pass of the formatter.  Please report a bug on https://github.com/psf/black/issues.  This diff might be helpful: /var/folders/cs/p77vt_fn271bcw6gw69d_2y40000gn/T/blk_i64a7rxi.log

ALSO

```sh
$ black fmt_skip.py . --fast
```

The resulting error is that the output is:
```python
import re

line = "some dummy data"
log_search = re.search('CLOCK:\s*(\[.*\])--(\[.*\])\s*=>\s*(\d*:\d*)', line)  #fmt:skip
  # fmt:skip  # fmt:skip
```

**Expected behavior**

Black should not change the line format as it is marked to be skipped.

**Environment**

* MacOS 12.2.1
* Python 3.8.5
* black 22.1.0

Also reproducible on the Black Playground: https://black.vercel.app/?version=main

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `src/black/comments.py`
Content:
```
1 import sys
2 from dataclasses import dataclass
3 from functools import lru_cache
4 import re
5 from typing import Iterator, List, Optional, Union
6 
7 if sys.version_info >= (3, 8):
8     from typing import Final
9 else:
10     from typing_extensions import Final
11 
12 from blib2to3.pytree import Node, Leaf
13 from blib2to3.pgen2 import token
14 
15 from black.nodes import first_leaf_column, preceding_leaf, container_of
16 from black.nodes import STANDALONE_COMMENT, WHITESPACE
17 
18 # types
19 LN = Union[Leaf, Node]
20 
21 FMT_OFF: Final = {"# fmt: off", "# fmt:off", "# yapf: disable"}
22 FMT_SKIP: Final = {"# fmt: skip", "# fmt:skip"}
23 FMT_PASS: Final = {*FMT_OFF, *FMT_SKIP}
24 FMT_ON: Final = {"# fmt: on", "# fmt:on", "# yapf: enable"}
25 
26 COMMENT_EXCEPTIONS = {True: " !:#'", False: " !:#'%"}
27 
28 
29 @dataclass
30 class ProtoComment:
31     """Describes a piece of syntax that is a comment.
32 
33     It's not a :class:`blib2to3.pytree.Leaf` so that:
34 
35     * it can be cached (`Leaf` objects should not be reused more than once as
36       they store their lineno, column, prefix, and parent information);
37     * `newlines` and `consumed` fields are kept separate from the `value`. This
38       simplifies handling of special marker comments like ``# fmt: off/on``.
39     """
40 
41     type: int  # token.COMMENT or STANDALONE_COMMENT
42     value: str  # content of the comment
43     newlines: int  # how many newlines before the comment
44     consumed: int  # how many characters of the original leaf's prefix did we consume
45 
46 
47 def generate_comments(leaf: LN, *, preview: bool) -> Iterator[Leaf]:
48     """Clean the prefix of the `leaf` and generate comments from it, if any.
49 
50     Comments in lib2to3 are shoved into the whitespace prefix.  This happens
51     in `pgen2/driver.py:Driver.parse_tokens()`.  This was a brilliant implementation
52     move because it does away with modifying the grammar to include all the
53     possible places in which comments can be placed.
54 
55     The sad consequence for us though is that comments don't "belong" anywhere.
56     This is why this function generates simple parentless Leaf objects for
57     comments.  We simply don't know what the correct parent should be.
58 
59     No matter though, we can live without this.  We really only need to
60     differentiate between inline and standalone comments.  The latter don't
61     share the line with any code.
62 
63     Inline comments are emitted as regular token.COMMENT leaves.  Standalone
64     are emitted with a fake STANDALONE_COMMENT token identifier.
65     """
66     for pc in list_comments(
67         leaf.prefix, is_endmarker=leaf.type == token.ENDMARKER, preview=preview
68     ):
69         yield Leaf(pc.type, pc.value, prefix="\n" * pc.newlines)
70 
71 
72 @lru_cache(maxsize=4096)
73 def list_comments(
74     prefix: str, *, is_endmarker: bool, preview: bool
75 ) -> List[ProtoComment]:
76     """Return a list of :class:`ProtoComment` objects parsed from the given `prefix`."""
77     result: List[ProtoComment] = []
78     if not prefix or "#" not in prefix:
79         return result
80 
81     consumed = 0
82     nlines = 0
83     ignored_lines = 0
84     for index, line in enumerate(re.split("\r?\n", prefix)):
85         consumed += len(line) + 1  # adding the length of the split '\n'
86         line = line.lstrip()
87         if not line:
88             nlines += 1
89         if not line.startswith("#"):
90             # Escaped newlines outside of a comment are not really newlines at
91             # all. We treat a single-line comment following an escaped newline
92             # as a simple trailing comment.
93             if line.endswith("\\"):
94                 ignored_lines += 1
95             continue
96 
97         if index == ignored_lines and not is_endmarker:
98             comment_type = token.COMMENT  # simple trailing comment
99         else:
100             comment_type = STANDALONE_COMMENT
101         comment = make_comment(line, preview=preview)
102         result.append(
103             ProtoComment(
104                 type=comment_type, value=comment, newlines=nlines, consumed=consumed
105             )
106         )
107         nlines = 0
108     return result
109 
110 
111 def make_comment(content: str, *, preview: bool) -> str:
112     """Return a consistently formatted comment from the given `content` string.
113 
114     All comments (except for "##", "#!", "#:", '#'") should have a single
115     space between the hash sign and the content.
116 
117     If `content` didn't start with a hash sign, one is provided.
118     """
119     content = content.rstrip()
120     if not content:
121         return "#"
122 
123     if content[0] == "#":
124         content = content[1:]
125     NON_BREAKING_SPACE = " "
126     if (
127         content
128         and content[0] == NON_BREAKING_SPACE
129         and not content.lstrip().startswith("type:")
130     ):
131         content = " " + content[1:]  # Replace NBSP by a simple space
132     if content and content[0] not in COMMENT_EXCEPTIONS[preview]:
133         content = " " + content
134     return "#" + content
135 
136 
137 def normalize_fmt_off(node: Node, *, preview: bool) -> None:
138     """Convert content between `# fmt: off`/`# fmt: on` into standalone comments."""
139     try_again = True
140     while try_again:
141         try_again = convert_one_fmt_off_pair(node, preview=preview)
142 
143 
144 def convert_one_fmt_off_pair(node: Node, *, preview: bool) -> bool:
145     """Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment.
146 
147     Returns True if a pair was converted.
148     """
149     for leaf in node.leaves():
150         previous_consumed = 0
151         for comment in list_comments(leaf.prefix, is_endmarker=False, preview=preview):
152             if comment.value not in FMT_PASS:
153                 previous_consumed = comment.consumed
154                 continue
155             # We only want standalone comments. If there's no previous leaf or
156             # the previous leaf is indentation, it's a standalone comment in
157             # disguise.
158             if comment.value in FMT_PASS and comment.type != STANDALONE_COMMENT:
159                 prev = preceding_leaf(leaf)
160                 if prev:
161                     if comment.value in FMT_OFF and prev.type not in WHITESPACE:
162                         continue
163                     if comment.value in FMT_SKIP and prev.type in WHITESPACE:
164                         continue
165 
166             ignored_nodes = list(generate_ignored_nodes(leaf, comment, preview=preview))
167             if not ignored_nodes:
168                 continue
169 
170             first = ignored_nodes[0]  # Can be a container node with the `leaf`.
171             parent = first.parent
172             prefix = first.prefix
173             if comment.value in FMT_OFF:
174                 first.prefix = prefix[comment.consumed :]
175             if comment.value in FMT_SKIP:
176                 first.prefix = ""
177             hidden_value = "".join(str(n) for n in ignored_nodes)
178             if comment.value in FMT_OFF:
179                 hidden_value = comment.value + "\n" + hidden_value
180             if comment.value in FMT_SKIP:
181                 hidden_value += "  " + comment.value
182             if hidden_value.endswith("\n"):
183                 # That happens when one of the `ignored_nodes` ended with a NEWLINE
184                 # leaf (possibly followed by a DEDENT).
185                 hidden_value = hidden_value[:-1]
186             first_idx: Optional[int] = None
187             for ignored in ignored_nodes:
188                 index = ignored.remove()
189                 if first_idx is None:
190                     first_idx = index
191             assert parent is not None, "INTERNAL ERROR: fmt: on/off handling (1)"
192             assert first_idx is not None, "INTERNAL ERROR: fmt: on/off handling (2)"
193             parent.insert_child(
194                 first_idx,
195                 Leaf(
196                     STANDALONE_COMMENT,
197                     hidden_value,
198                     prefix=prefix[:previous_consumed] + "\n" * comment.newlines,
199                 ),
200             )
201             return True
202 
203     return False
204 
205 
206 def generate_ignored_nodes(
207     leaf: Leaf, comment: ProtoComment, *, preview: bool
208 ) -> Iterator[LN]:
209     """Starting from the container of `leaf`, generate all leaves until `# fmt: on`.
210 
211     If comment is skip, returns leaf only.
212     Stops at the end of the block.
213     """
214     container: Optional[LN] = container_of(leaf)
215     if comment.value in FMT_SKIP:
216         prev_sibling = leaf.prev_sibling
217         if comment.value in leaf.prefix and prev_sibling is not None:
218             leaf.prefix = leaf.prefix.replace(comment.value, "")
219             siblings = [prev_sibling]
220             while (
221                 "\n" not in prev_sibling.prefix
222                 and prev_sibling.prev_sibling is not None
223             ):
224                 prev_sibling = prev_sibling.prev_sibling
225                 siblings.insert(0, prev_sibling)
226             for sibling in siblings:
227                 yield sibling
228         elif leaf.parent is not None:
229             yield leaf.parent
230         return
231     while container is not None and container.type != token.ENDMARKER:
232         if is_fmt_on(container, preview=preview):
233             return
234 
235         # fix for fmt: on in children
236         if contains_fmt_on_at_column(container, leaf.column, preview=preview):
237             for child in container.children:
238                 if contains_fmt_on_at_column(child, leaf.column, preview=preview):
239                     return
240                 yield child
241         else:
242             yield container
243             container = container.next_sibling
244 
245 
246 def is_fmt_on(container: LN, preview: bool) -> bool:
247     """Determine whether formatting is switched on within a container.
248     Determined by whether the last `# fmt:` comment is `on` or `off`.
249     """
250     fmt_on = False
251     for comment in list_comments(container.prefix, is_endmarker=False, preview=preview):
252         if comment.value in FMT_ON:
253             fmt_on = True
254         elif comment.value in FMT_OFF:
255             fmt_on = False
256     return fmt_on
257 
258 
259 def contains_fmt_on_at_column(container: LN, column: int, *, preview: bool) -> bool:
260     """Determine if children at a given column have formatting switched on."""
261     for child in container.children:
262         if (
263             isinstance(child, Node)
264             and first_leaf_column(child) == column
265             or isinstance(child, Leaf)
266             and child.column == column
267         ):
268             if is_fmt_on(child, preview=preview):
269                 return True
270 
271     return False
272 
273 
274 def contains_pragma_comment(comment_list: List[Leaf]) -> bool:
275     """
276     Returns:
277         True iff one of the comments in @comment_list is a pragma used by one
278         of the more common static analysis tools for python (e.g. mypy, flake8,
279         pylint).
280     """
281     for comment in comment_list:
282         if comment.value.startswith(("# type:", "# noqa", "# pylint:")):
283             return True
284 
285     return False
286
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/src/black/comments.py b/src/black/comments.py
--- a/src/black/comments.py
+++ b/src/black/comments.py
@@ -214,8 +214,11 @@
     container: Optional[LN] = container_of(leaf)
     if comment.value in FMT_SKIP:
         prev_sibling = leaf.prev_sibling
-        if comment.value in leaf.prefix and prev_sibling is not None:
-            leaf.prefix = leaf.prefix.replace(comment.value, "")
+        # Need to properly format the leaf prefix to compare it to comment.value,
+        # which is also formatted
+        comments = list_comments(leaf.prefix, is_endmarker=False, preview=preview)
+        if comments and comment.value == comments[0].value and prev_sibling is not None:
+            leaf.prefix = ""
             siblings = [prev_sibling]
             while (
                 "\n" not in prev_sibling.prefix

{"golden_diff": "diff --git a/src/black/comments.py b/src/black/comments.py\n--- a/src/black/comments.py\n+++ b/src/black/comments.py\n@@ -214,8 +214,11 @@\n     container: Optional[LN] = container_of(leaf)\n     if comment.value in FMT_SKIP:\n         prev_sibling = leaf.prev_sibling\n-        if comment.value in leaf.prefix and prev_sibling is not None:\n-            leaf.prefix = leaf.prefix.replace(comment.value, \"\")\n+        # Need to properly format the leaf prefix to compare it to comment.value,\n+        # which is also formatted\n+        comments = list_comments(leaf.prefix, is_endmarker=False, preview=preview)\n+        if comments and comment.value == comments[0].value and prev_sibling is not None:\n+            leaf.prefix = \"\"\n             siblings = [prev_sibling]\n             while (\n                 \"\\n\" not in prev_sibling.prefix\n", "issue": "`#fmt:skip` causes NTERNAL ERROR: Black produced different code on the second pass of the formatter\n**Describe the bug**\r\n\r\nA comment `#fmt:skip` with no internal space causes `INTERNAL ERROR: Black produced different code on the second pass of the formatter`\r\n\r\nAlso the same code, if run with `--fast` creates incorrectly formatted code with multiple copies of the comment on the next line.\r\n\r\n**To Reproduce**\r\n\r\nFor example, take this code:\r\n\r\n```python\r\nimport re\r\n\r\nline = \"some dummy data\"\r\nlog_search = re.search('CLOCK:\\s*(\\[.*\\])--(\\[.*\\])\\s*=>\\s*(\\d*:\\d*)', line)  #fmt:skip\r\n```\r\n\r\nAnd run it with these arguments:\r\n\r\n```sh\r\n$ black fmt_skip.py .\r\n```\r\n\r\nThe resulting error is:\r\n\r\nerror: cannot format fmt_skip.py: INTERNAL ERROR: Black produced different code on the second pass of the formatter.  Please report a bug on https://github.com/psf/black/issues.  This diff might be helpful: /var/folders/cs/p77vt_fn271bcw6gw69d_2y40000gn/T/blk_i64a7rxi.log\r\n\r\nALSO\r\n\r\n```sh\r\n$ black fmt_skip.py . --fast\r\n```\r\n\r\nThe resulting error is that the output is:\r\n```python\r\nimport re\r\n\r\nline = \"some dummy data\"\r\nlog_search = re.search('CLOCK:\\s*(\\[.*\\])--(\\[.*\\])\\s*=>\\s*(\\d*:\\d*)', line)  #fmt:skip\r\n  # fmt:skip  # fmt:skip\r\n```\r\n\r\n**Expected behavior**\r\n\r\nBlack should not change the line format as it is marked to be skipped.\r\n\r\n**Environment**\r\n\r\n* MacOS 12.2.1\r\n* Python 3.8.5\r\n* black 22.1.0\r\n\r\nAlso reproducible on the Black Playground: https://black.vercel.app/?version=main\n", "before_files": [{"content": "import sys\nfrom dataclasses import dataclass\nfrom functools import lru_cache\nimport re\nfrom typing import Iterator, List, Optional, Union\n\nif sys.version_info >= (3, 8):\n    from typing import Final\nelse:\n    from typing_extensions import Final\n\nfrom blib2to3.pytree import Node, Leaf\nfrom blib2to3.pgen2 import token\n\nfrom black.nodes import first_leaf_column, preceding_leaf, container_of\nfrom black.nodes import STANDALONE_COMMENT, WHITESPACE\n\n# types\nLN = Union[Leaf, Node]\n\nFMT_OFF: Final = {\"# fmt: off\", \"# fmt:off\", \"# yapf: disable\"}\nFMT_SKIP: Final = {\"# fmt: skip\", \"# fmt:skip\"}\nFMT_PASS: Final = {*FMT_OFF, *FMT_SKIP}\nFMT_ON: Final = {\"# fmt: on\", \"# fmt:on\", \"# yapf: enable\"}\n\nCOMMENT_EXCEPTIONS = {True: \" !:#'\", False: \" !:#'%\"}\n\n\n@dataclass\nclass ProtoComment:\n    \"\"\"Describes a piece of syntax that is a comment.\n\n    It's not a :class:`blib2to3.pytree.Leaf` so that:\n\n    * it can be cached (`Leaf` objects should not be reused more than once as\n      they store their lineno, column, prefix, and parent information);\n    * `newlines` and `consumed` fields are kept separate from the `value`. This\n      simplifies handling of special marker comments like ``# fmt: off/on``.\n    \"\"\"\n\n    type: int  # token.COMMENT or STANDALONE_COMMENT\n    value: str  # content of the comment\n    newlines: int  # how many newlines before the comment\n    consumed: int  # how many characters of the original leaf's prefix did we consume\n\n\ndef generate_comments(leaf: LN, *, preview: bool) -> Iterator[Leaf]:\n    \"\"\"Clean the prefix of the `leaf` and generate comments from it, if any.\n\n    Comments in lib2to3 are shoved into the whitespace prefix.  This happens\n    in `pgen2/driver.py:Driver.parse_tokens()`.  This was a brilliant implementation\n    move because it does away with modifying the grammar to include all the\n    possible places in which comments can be placed.\n\n    The sad consequence for us though is that comments don't \"belong\" anywhere.\n    This is why this function generates simple parentless Leaf objects for\n    comments.  We simply don't know what the correct parent should be.\n\n    No matter though, we can live without this.  We really only need to\n    differentiate between inline and standalone comments.  The latter don't\n    share the line with any code.\n\n    Inline comments are emitted as regular token.COMMENT leaves.  Standalone\n    are emitted with a fake STANDALONE_COMMENT token identifier.\n    \"\"\"\n    for pc in list_comments(\n        leaf.prefix, is_endmarker=leaf.type == token.ENDMARKER, preview=preview\n    ):\n        yield Leaf(pc.type, pc.value, prefix=\"\\n\" * pc.newlines)\n\n\n@lru_cache(maxsize=4096)\ndef list_comments(\n    prefix: str, *, is_endmarker: bool, preview: bool\n) -> List[ProtoComment]:\n    \"\"\"Return a list of :class:`ProtoComment` objects parsed from the given `prefix`.\"\"\"\n    result: List[ProtoComment] = []\n    if not prefix or \"#\" not in prefix:\n        return result\n\n    consumed = 0\n    nlines = 0\n    ignored_lines = 0\n    for index, line in enumerate(re.split(\"\\r?\\n\", prefix)):\n        consumed += len(line) + 1  # adding the length of the split '\\n'\n        line = line.lstrip()\n        if not line:\n            nlines += 1\n        if not line.startswith(\"#\"):\n            # Escaped newlines outside of a comment are not really newlines at\n            # all. We treat a single-line comment following an escaped newline\n            # as a simple trailing comment.\n            if line.endswith(\"\\\\\"):\n                ignored_lines += 1\n            continue\n\n        if index == ignored_lines and not is_endmarker:\n            comment_type = token.COMMENT  # simple trailing comment\n        else:\n            comment_type = STANDALONE_COMMENT\n        comment = make_comment(line, preview=preview)\n        result.append(\n            ProtoComment(\n                type=comment_type, value=comment, newlines=nlines, consumed=consumed\n            )\n        )\n        nlines = 0\n    return result\n\n\ndef make_comment(content: str, *, preview: bool) -> str:\n    \"\"\"Return a consistently formatted comment from the given `content` string.\n\n    All comments (except for \"##\", \"#!\", \"#:\", '#'\") should have a single\n    space between the hash sign and the content.\n\n    If `content` didn't start with a hash sign, one is provided.\n    \"\"\"\n    content = content.rstrip()\n    if not content:\n        return \"#\"\n\n    if content[0] == \"#\":\n        content = content[1:]\n    NON_BREAKING_SPACE = \"\u00a0\"\n    if (\n        content\n        and content[0] == NON_BREAKING_SPACE\n        and not content.lstrip().startswith(\"type:\")\n    ):\n        content = \" \" + content[1:]  # Replace NBSP by a simple space\n    if content and content[0] not in COMMENT_EXCEPTIONS[preview]:\n        content = \" \" + content\n    return \"#\" + content\n\n\ndef normalize_fmt_off(node: Node, *, preview: bool) -> None:\n    \"\"\"Convert content between `# fmt: off`/`# fmt: on` into standalone comments.\"\"\"\n    try_again = True\n    while try_again:\n        try_again = convert_one_fmt_off_pair(node, preview=preview)\n\n\ndef convert_one_fmt_off_pair(node: Node, *, preview: bool) -> bool:\n    \"\"\"Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment.\n\n    Returns True if a pair was converted.\n    \"\"\"\n    for leaf in node.leaves():\n        previous_consumed = 0\n        for comment in list_comments(leaf.prefix, is_endmarker=False, preview=preview):\n            if comment.value not in FMT_PASS:\n                previous_consumed = comment.consumed\n                continue\n            # We only want standalone comments. If there's no previous leaf or\n            # the previous leaf is indentation, it's a standalone comment in\n            # disguise.\n            if comment.value in FMT_PASS and comment.type != STANDALONE_COMMENT:\n                prev = preceding_leaf(leaf)\n                if prev:\n                    if comment.value in FMT_OFF and prev.type not in WHITESPACE:\n                        continue\n                    if comment.value in FMT_SKIP and prev.type in WHITESPACE:\n                        continue\n\n            ignored_nodes = list(generate_ignored_nodes(leaf, comment, preview=preview))\n            if not ignored_nodes:\n                continue\n\n            first = ignored_nodes[0]  # Can be a container node with the `leaf`.\n            parent = first.parent\n            prefix = first.prefix\n            if comment.value in FMT_OFF:\n                first.prefix = prefix[comment.consumed :]\n            if comment.value in FMT_SKIP:\n                first.prefix = \"\"\n            hidden_value = \"\".join(str(n) for n in ignored_nodes)\n            if comment.value in FMT_OFF:\n                hidden_value = comment.value + \"\\n\" + hidden_value\n            if comment.value in FMT_SKIP:\n                hidden_value += \"  \" + comment.value\n            if hidden_value.endswith(\"\\n\"):\n                # That happens when one of the `ignored_nodes` ended with a NEWLINE\n                # leaf (possibly followed by a DEDENT).\n                hidden_value = hidden_value[:-1]\n            first_idx: Optional[int] = None\n            for ignored in ignored_nodes:\n                index = ignored.remove()\n                if first_idx is None:\n                    first_idx = index\n            assert parent is not None, \"INTERNAL ERROR: fmt: on/off handling (1)\"\n            assert first_idx is not None, \"INTERNAL ERROR: fmt: on/off handling (2)\"\n            parent.insert_child(\n                first_idx,\n                Leaf(\n                    STANDALONE_COMMENT,\n                    hidden_value,\n                    prefix=prefix[:previous_consumed] + \"\\n\" * comment.newlines,\n                ),\n            )\n            return True\n\n    return False\n\n\ndef generate_ignored_nodes(\n    leaf: Leaf, comment: ProtoComment, *, preview: bool\n) -> Iterator[LN]:\n    \"\"\"Starting from the container of `leaf`, generate all leaves until `# fmt: on`.\n\n    If comment is skip, returns leaf only.\n    Stops at the end of the block.\n    \"\"\"\n    container: Optional[LN] = container_of(leaf)\n    if comment.value in FMT_SKIP:\n        prev_sibling = leaf.prev_sibling\n        if comment.value in leaf.prefix and prev_sibling is not None:\n            leaf.prefix = leaf.prefix.replace(comment.value, \"\")\n            siblings = [prev_sibling]\n            while (\n                \"\\n\" not in prev_sibling.prefix\n                and prev_sibling.prev_sibling is not None\n            ):\n                prev_sibling = prev_sibling.prev_sibling\n                siblings.insert(0, prev_sibling)\n            for sibling in siblings:\n                yield sibling\n        elif leaf.parent is not None:\n            yield leaf.parent\n        return\n    while container is not None and container.type != token.ENDMARKER:\n        if is_fmt_on(container, preview=preview):\n            return\n\n        # fix for fmt: on in children\n        if contains_fmt_on_at_column(container, leaf.column, preview=preview):\n            for child in container.children:\n                if contains_fmt_on_at_column(child, leaf.column, preview=preview):\n                    return\n                yield child\n        else:\n            yield container\n            container = container.next_sibling\n\n\ndef is_fmt_on(container: LN, preview: bool) -> bool:\n    \"\"\"Determine whether formatting is switched on within a container.\n    Determined by whether the last `# fmt:` comment is `on` or `off`.\n    \"\"\"\n    fmt_on = False\n    for comment in list_comments(container.prefix, is_endmarker=False, preview=preview):\n        if comment.value in FMT_ON:\n            fmt_on = True\n        elif comment.value in FMT_OFF:\n            fmt_on = False\n    return fmt_on\n\n\ndef contains_fmt_on_at_column(container: LN, column: int, *, preview: bool) -> bool:\n    \"\"\"Determine if children at a given column have formatting switched on.\"\"\"\n    for child in container.children:\n        if (\n            isinstance(child, Node)\n            and first_leaf_column(child) == column\n            or isinstance(child, Leaf)\n            and child.column == column\n        ):\n            if is_fmt_on(child, preview=preview):\n                return True\n\n    return False\n\n\ndef contains_pragma_comment(comment_list: List[Leaf]) -> bool:\n    \"\"\"\n    Returns:\n        True iff one of the comments in @comment_list is a pragma used by one\n        of the more common static analysis tools for python (e.g. mypy, flake8,\n        pylint).\n    \"\"\"\n    for comment in comment_list:\n        if comment.value.startswith((\"# type:\", \"# noqa\", \"# pylint:\")):\n            return True\n\n    return False\n", "path": "src/black/comments.py"}], "after_files": [{"content": "import sys\nfrom dataclasses import dataclass\nfrom functools import lru_cache\nimport re\nfrom typing import Iterator, List, Optional, Union\n\nif sys.version_info >= (3, 8):\n    from typing import Final\nelse:\n    from typing_extensions import Final\n\nfrom blib2to3.pytree import Node, Leaf\nfrom blib2to3.pgen2 import token\n\nfrom black.nodes import first_leaf_column, preceding_leaf, container_of\nfrom black.nodes import STANDALONE_COMMENT, WHITESPACE\n\n# types\nLN = Union[Leaf, Node]\n\nFMT_OFF: Final = {\"# fmt: off\", \"# fmt:off\", \"# yapf: disable\"}\nFMT_SKIP: Final = {\"# fmt: skip\", \"# fmt:skip\"}\nFMT_PASS: Final = {*FMT_OFF, *FMT_SKIP}\nFMT_ON: Final = {\"# fmt: on\", \"# fmt:on\", \"# yapf: enable\"}\n\nCOMMENT_EXCEPTIONS = {True: \" !:#'\", False: \" !:#'%\"}\n\n\n@dataclass\nclass ProtoComment:\n    \"\"\"Describes a piece of syntax that is a comment.\n\n    It's not a :class:`blib2to3.pytree.Leaf` so that:\n\n    * it can be cached (`Leaf` objects should not be reused more than once as\n      they store their lineno, column, prefix, and parent information);\n    * `newlines` and `consumed` fields are kept separate from the `value`. This\n      simplifies handling of special marker comments like ``# fmt: off/on``.\n    \"\"\"\n\n    type: int  # token.COMMENT or STANDALONE_COMMENT\n    value: str  # content of the comment\n    newlines: int  # how many newlines before the comment\n    consumed: int  # how many characters of the original leaf's prefix did we consume\n\n\ndef generate_comments(leaf: LN, *, preview: bool) -> Iterator[Leaf]:\n    \"\"\"Clean the prefix of the `leaf` and generate comments from it, if any.\n\n    Comments in lib2to3 are shoved into the whitespace prefix.  This happens\n    in `pgen2/driver.py:Driver.parse_tokens()`.  This was a brilliant implementation\n    move because it does away with modifying the grammar to include all the\n    possible places in which comments can be placed.\n\n    The sad consequence for us though is that comments don't \"belong\" anywhere.\n    This is why this function generates simple parentless Leaf objects for\n    comments.  We simply don't know what the correct parent should be.\n\n    No matter though, we can live without this.  We really only need to\n    differentiate between inline and standalone comments.  The latter don't\n    share the line with any code.\n\n    Inline comments are emitted as regular token.COMMENT leaves.  Standalone\n    are emitted with a fake STANDALONE_COMMENT token identifier.\n    \"\"\"\n    for pc in list_comments(\n        leaf.prefix, is_endmarker=leaf.type == token.ENDMARKER, preview=preview\n    ):\n        yield Leaf(pc.type, pc.value, prefix=\"\\n\" * pc.newlines)\n\n\n@lru_cache(maxsize=4096)\ndef list_comments(\n    prefix: str, *, is_endmarker: bool, preview: bool\n) -> List[ProtoComment]:\n    \"\"\"Return a list of :class:`ProtoComment` objects parsed from the given `prefix`.\"\"\"\n    result: List[ProtoComment] = []\n    if not prefix or \"#\" not in prefix:\n        return result\n\n    consumed = 0\n    nlines = 0\n    ignored_lines = 0\n    for index, line in enumerate(re.split(\"\\r?\\n\", prefix)):\n        consumed += len(line) + 1  # adding the length of the split '\\n'\n        line = line.lstrip()\n        if not line:\n            nlines += 1\n        if not line.startswith(\"#\"):\n            # Escaped newlines outside of a comment are not really newlines at\n            # all. We treat a single-line comment following an escaped newline\n            # as a simple trailing comment.\n            if line.endswith(\"\\\\\"):\n                ignored_lines += 1\n            continue\n\n        if index == ignored_lines and not is_endmarker:\n            comment_type = token.COMMENT  # simple trailing comment\n        else:\n            comment_type = STANDALONE_COMMENT\n        comment = make_comment(line, preview=preview)\n        result.append(\n            ProtoComment(\n                type=comment_type, value=comment, newlines=nlines, consumed=consumed\n            )\n        )\n        nlines = 0\n    return result\n\n\ndef make_comment(content: str, *, preview: bool) -> str:\n    \"\"\"Return a consistently formatted comment from the given `content` string.\n\n    All comments (except for \"##\", \"#!\", \"#:\", '#'\") should have a single\n    space between the hash sign and the content.\n\n    If `content` didn't start with a hash sign, one is provided.\n    \"\"\"\n    content = content.rstrip()\n    if not content:\n        return \"#\"\n\n    if content[0] == \"#\":\n        content = content[1:]\n    NON_BREAKING_SPACE = \"\u00a0\"\n    if (\n        content\n        and content[0] == NON_BREAKING_SPACE\n        and not content.lstrip().startswith(\"type:\")\n    ):\n        content = \" \" + content[1:]  # Replace NBSP by a simple space\n    if content and content[0] not in COMMENT_EXCEPTIONS[preview]:\n        content = \" \" + content\n    return \"#\" + content\n\n\ndef normalize_fmt_off(node: Node, *, preview: bool) -> None:\n    \"\"\"Convert content between `# fmt: off`/`# fmt: on` into standalone comments.\"\"\"\n    try_again = True\n    while try_again:\n        try_again = convert_one_fmt_off_pair(node, preview=preview)\n\n\ndef convert_one_fmt_off_pair(node: Node, *, preview: bool) -> bool:\n    \"\"\"Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment.\n\n    Returns True if a pair was converted.\n    \"\"\"\n    for leaf in node.leaves():\n        previous_consumed = 0\n        for comment in list_comments(leaf.prefix, is_endmarker=False, preview=preview):\n            if comment.value not in FMT_PASS:\n                previous_consumed = comment.consumed\n                continue\n            # We only want standalone comments. If there's no previous leaf or\n            # the previous leaf is indentation, it's a standalone comment in\n            # disguise.\n            if comment.value in FMT_PASS and comment.type != STANDALONE_COMMENT:\n                prev = preceding_leaf(leaf)\n                if prev:\n                    if comment.value in FMT_OFF and prev.type not in WHITESPACE:\n                        continue\n                    if comment.value in FMT_SKIP and prev.type in WHITESPACE:\n                        continue\n\n            ignored_nodes = list(generate_ignored_nodes(leaf, comment, preview=preview))\n            if not ignored_nodes:\n                continue\n\n            first = ignored_nodes[0]  # Can be a container node with the `leaf`.\n            parent = first.parent\n            prefix = first.prefix\n            if comment.value in FMT_OFF:\n                first.prefix = prefix[comment.consumed :]\n            if comment.value in FMT_SKIP:\n                first.prefix = \"\"\n            hidden_value = \"\".join(str(n) for n in ignored_nodes)\n            if comment.value in FMT_OFF:\n                hidden_value = comment.value + \"\\n\" + hidden_value\n            if comment.value in FMT_SKIP:\n                hidden_value += \"  \" + comment.value\n            if hidden_value.endswith(\"\\n\"):\n                # That happens when one of the `ignored_nodes` ended with a NEWLINE\n                # leaf (possibly followed by a DEDENT).\n                hidden_value = hidden_value[:-1]\n            first_idx: Optional[int] = None\n            for ignored in ignored_nodes:\n                index = ignored.remove()\n                if first_idx is None:\n                    first_idx = index\n            assert parent is not None, \"INTERNAL ERROR: fmt: on/off handling (1)\"\n            assert first_idx is not None, \"INTERNAL ERROR: fmt: on/off handling (2)\"\n            parent.insert_child(\n                first_idx,\n                Leaf(\n                    STANDALONE_COMMENT,\n                    hidden_value,\n                    prefix=prefix[:previous_consumed] + \"\\n\" * comment.newlines,\n                ),\n            )\n            return True\n\n    return False\n\n\ndef generate_ignored_nodes(\n    leaf: Leaf, comment: ProtoComment, *, preview: bool\n) -> Iterator[LN]:\n    \"\"\"Starting from the container of `leaf`, generate all leaves until `# fmt: on`.\n\n    If comment is skip, returns leaf only.\n    Stops at the end of the block.\n    \"\"\"\n    container: Optional[LN] = container_of(leaf)\n    if comment.value in FMT_SKIP:\n        prev_sibling = leaf.prev_sibling\n        # Need to properly format the leaf prefix to compare it to comment.value,\n        # which is also formatted\n        comments = list_comments(leaf.prefix, is_endmarker=False, preview=preview)\n        if comments and comment.value == comments[0].value and prev_sibling is not None:\n            leaf.prefix = \"\"\n            siblings = [prev_sibling]\n            while (\n                \"\\n\" not in prev_sibling.prefix\n                and prev_sibling.prev_sibling is not None\n            ):\n                prev_sibling = prev_sibling.prev_sibling\n                siblings.insert(0, prev_sibling)\n            for sibling in siblings:\n                yield sibling\n        elif leaf.parent is not None:\n            yield leaf.parent\n        return\n    while container is not None and container.type != token.ENDMARKER:\n        if is_fmt_on(container, preview=preview):\n            return\n\n        # fix for fmt: on in children\n        if contains_fmt_on_at_column(container, leaf.column, preview=preview):\n            for child in container.children:\n                if contains_fmt_on_at_column(child, leaf.column, preview=preview):\n                    return\n                yield child\n        else:\n            yield container\n            container = container.next_sibling\n\n\ndef is_fmt_on(container: LN, preview: bool) -> bool:\n    \"\"\"Determine whether formatting is switched on within a container.\n    Determined by whether the last `# fmt:` comment is `on` or `off`.\n    \"\"\"\n    fmt_on = False\n    for comment in list_comments(container.prefix, is_endmarker=False, preview=preview):\n        if comment.value in FMT_ON:\n            fmt_on = True\n        elif comment.value in FMT_OFF:\n            fmt_on = False\n    return fmt_on\n\n\ndef contains_fmt_on_at_column(container: LN, column: int, *, preview: bool) -> bool:\n    \"\"\"Determine if children at a given column have formatting switched on.\"\"\"\n    for child in container.children:\n        if (\n            isinstance(child, Node)\n            and first_leaf_column(child) == column\n            or isinstance(child, Leaf)\n            and child.column == column\n        ):\n            if is_fmt_on(child, preview=preview):\n                return True\n\n    return False\n\n\ndef contains_pragma_comment(comment_list: List[Leaf]) -> bool:\n    \"\"\"\n    Returns:\n        True iff one of the comments in @comment_list is a pragma used by one\n        of the more common static analysis tools for python (e.g. mypy, flake8,\n        pylint).\n    \"\"\"\n    for comment in comment_list:\n        if comment.value.startswith((\"# type:\", \"# noqa\", \"# pylint:\")):\n            return True\n\n    return False\n", "path": "src/black/comments.py"}]}