We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
wsc get_train_examples doesn't match wsc data format
In jiant/jiant/tasks/lib/wsc.py, `_create_examples()` dies at the statement `span1_idx=line["span1_index"]` (line 177) with `KeyError: 'span1_index'` because it mismatches the structure of the JSON task data.

The statement should be `span1_idx=line["target"]["span1_index"]` and similarly for the next 3 statements.

**To Reproduce**
1. Install jiant v2 from any recent commit (where wsc.py hasn't been touched since 3bd801ce)
2. I doubt this matters, but running Python version 3.8 in a recent linux on a 40 core, 80 thread skylake CPU with 384 GB of RAM and a VT100/16GB GPU.
3. In the Python REPL,
```python
from jiant.proj.simple import runscript as run
import jiant.scripts.download_data.runscript as downloader
downloader.download_data(["wsc"], "/home/rasmussen.63/wsc-speed/tasks")
args = run.RunConfiguration(
   run_name="wsc-speed",
   exp_dir="wsc-speed",
   data_dir="wsc-speed/tasks",
   model_type="roberta-base",
   tasks="wsc",
   train_batch_size=16,
   num_train_epochs=3
)
run.run_simple(args)
```
Watch stderr come back to you until it stops at

```
Tokenizing Task 'wsc' for phases 'train,val,test'
WSCTask
  [train]: /home/rasmussen.63/wsc-speed/tasks/data/wsc/train.jsonl
  [val]: /home/rasmussen.63/wsc-speed/tasks/data/wsc/val.jsonl
  [test]: /home/rasmussen.63/wsc-speed/tasks/data/wsc/test.jsonl
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/rasmussen.63/jiant/jiant/proj/simple/runscript.py", line 148, in run_simple
    tokenize_and_cache.main(
  File "/home/rasmussen.63/jiant/jiant/proj/main/tokenize_and_cache.py", line 165, in main
    examples=task.get_train_examples(),
  File "/home/rasmussen.63/jiant/jiant/tasks/lib/wsc.py", line 160, in get_train_examples
    return self._create_examples(lines=read_json_lines(self.train_path), set_type="train")
  File "/home/rasmussen.63/jiant/jiant/tasks/lib/wsc.py", line 177, in _create_examples
    span1_idx=line["span1_index"],
KeyError: 'span1_index'
```

**Expected behavior**
WSCTask should initialize an Example from the downloaded data.



--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `jiant/tasks/lib/wsc.py`
Content:
```
1 import numpy as np
2 import torch
3 from dataclasses import dataclass
4 from typing import List
5 
6 from jiant.tasks.core import (
7     BaseExample,
8     BaseTokenizedExample,
9     BaseDataRow,
10     BatchMixin,
11     SuperGlueMixin,
12     Task,
13     TaskTypes,
14 )
15 from jiant.tasks.lib.templates.shared import (
16     labels_to_bimap,
17     add_cls_token,
18     create_input_set_from_tokens_and_segments,
19 )
20 from jiant.tasks.utils import truncate_sequences, ExclusiveSpan
21 from jiant.utils import retokenize
22 from jiant.utils.python.io import read_json_lines
23 from jiant.utils.tokenization_normalization import normalize_tokenizations
24 
25 
26 @dataclass
27 class Example(BaseExample):
28     guid: str
29     text: str
30     span1_idx: int
31     span2_idx: int
32     span1_text: str
33     span2_text: str
34     label: str
35 
36     def tokenize(self, tokenizer):
37         space_tokenization = self.text.split()
38         target_tokenization = tokenizer.tokenize(self.text)
39         normed_space_tokenization, normed_target_tokenization = normalize_tokenizations(
40             space_tokenization, target_tokenization, tokenizer
41         )
42         aligner = retokenize.TokenAligner(normed_space_tokenization, normed_target_tokenization)
43         span1_token_count = len(self.span1_text.split())
44         span2_token_count = len(self.span2_text.split())
45         target_span1 = ExclusiveSpan(
46             *aligner.project_token_span(self.span1_idx, self.span1_idx + span1_token_count)
47         )
48         target_span2 = ExclusiveSpan(
49             *aligner.project_token_span(self.span2_idx, self.span2_idx + span2_token_count)
50         )
51         return TokenizedExample(
52             guid=self.guid,
53             tokens=target_tokenization,
54             span1_span=target_span1,
55             span2_span=target_span2,
56             span1_text=self.span1_text,
57             span2_text=self.span2_text,
58             label_id=WSCTask.LABEL_TO_ID[self.label],
59         )
60 
61 
62 @dataclass
63 class TokenizedExample(BaseTokenizedExample):
64     guid: str
65     tokens: List
66     span1_span: ExclusiveSpan
67     span2_span: ExclusiveSpan
68     span1_text: str
69     span2_text: str
70     label_id: int
71 
72     def featurize(self, tokenizer, feat_spec):
73         special_tokens_count = 2  # CLS, SEP
74 
75         (tokens,) = truncate_sequences(
76             tokens_ls=[self.tokens], max_length=feat_spec.max_seq_length - special_tokens_count,
77         )
78 
79         unpadded_tokens = tokens + [tokenizer.sep_token]
80         unpadded_segment_ids = [feat_spec.sequence_a_segment_id] * (len(self.tokens) + 1)
81 
82         unpadded_inputs = add_cls_token(
83             unpadded_tokens=unpadded_tokens,
84             unpadded_segment_ids=unpadded_segment_ids,
85             tokenizer=tokenizer,
86             feat_spec=feat_spec,
87         )
88 
89         input_set = create_input_set_from_tokens_and_segments(
90             unpadded_tokens=unpadded_inputs.unpadded_tokens,
91             unpadded_segment_ids=unpadded_inputs.unpadded_segment_ids,
92             tokenizer=tokenizer,
93             feat_spec=feat_spec,
94         )
95         span1_span = ExclusiveSpan(
96             start=self.span1_span[0] + unpadded_inputs.cls_offset,
97             end=self.span1_span[1] + unpadded_inputs.cls_offset,
98         ).to_inclusive()
99         span2_span = ExclusiveSpan(
100             start=self.span2_span[0] + unpadded_inputs.cls_offset,
101             end=self.span2_span[1] + unpadded_inputs.cls_offset,
102         ).to_inclusive()
103 
104         return DataRow(
105             guid=self.guid,
106             input_ids=np.array(input_set.input_ids),
107             input_mask=np.array(input_set.input_mask),
108             segment_ids=np.array(input_set.segment_ids),
109             spans=np.array([span1_span, span2_span]),
110             label_id=self.label_id,
111             tokens=unpadded_inputs.unpadded_tokens,
112             span1_text=self.span1_text,
113             span2_text=self.span2_text,
114         )
115 
116 
117 @dataclass
118 class DataRow(BaseDataRow):
119     guid: str
120     input_ids: np.ndarray
121     input_mask: np.ndarray
122     segment_ids: np.ndarray
123     spans: np.ndarray
124     label_id: int
125     tokens: List
126     span1_text: str
127     span2_text: str
128 
129     def get_tokens(self):
130         return [self.tokens]
131 
132 
133 @dataclass
134 class Batch(BatchMixin):
135     input_ids: torch.LongTensor
136     input_mask: torch.LongTensor
137     segment_ids: torch.LongTensor
138     spans: torch.LongTensor
139     label_id: torch.LongTensor
140     tokens: List
141     span1_text: List
142     span2_text: List
143 
144 
145 class WSCTask(SuperGlueMixin, Task):
146     Example = Example
147     TokenizedExample = Example
148     DataRow = DataRow
149     Batch = Batch
150 
151     TASK_TYPE = TaskTypes.SPAN_COMPARISON_CLASSIFICATION
152     LABELS = [False, True]
153     LABEL_TO_ID, ID_TO_LABEL = labels_to_bimap(LABELS)
154 
155     @property
156     def num_spans(self):
157         return 2
158 
159     def get_train_examples(self):
160         return self._create_examples(lines=read_json_lines(self.train_path), set_type="train")
161 
162     def get_val_examples(self):
163         return self._create_examples(lines=read_json_lines(self.val_path), set_type="val")
164 
165     def get_test_examples(self):
166         return self._create_examples(lines=read_json_lines(self.test_path), set_type="test")
167 
168     @classmethod
169     def _create_examples(cls, lines, set_type):
170         examples = []
171         for line in lines:
172             examples.append(
173                 Example(
174                     # NOTE: WSCTask.super_glue_format_preds() is dependent on this guid format.
175                     guid="%s-%s" % (set_type, line["idx"]),
176                     text=line["text"],
177                     span1_idx=line["span1_index"],
178                     span2_idx=line["span2_index"],
179                     span1_text=line["span1_text"],
180                     span2_text=line["span2_text"],
181                     label=line["label"] if set_type != "test" else cls.LABELS[-1],
182                 )
183             )
184         return examples
185 
186     @classmethod
187     def super_glue_format_preds(cls, pred_dict):
188         """Reformat this task's raw predictions to have the structure expected by SuperGLUE."""
189         lines = []
190         for pred, guid in zip(list(pred_dict["preds"]), list(pred_dict["guids"])):
191             lines.append({"idx": int(guid.split("-")[1]), "label": str(cls.LABELS[pred])})
192         return lines
193 
194 
195 def extract_char_span(full_text, span_text, space_index):
196     space_tokens = full_text.split()
197     extracted_span_text = space_tokens[space_index]
198     assert extracted_span_text.lower() in full_text.lower()
199     span_length = len(span_text)
200     if space_index == 0:
201         start = 0
202     else:
203         start = len(" ".join(space_tokens[:space_index])) + 1
204     # exclusive span
205     return ExclusiveSpan(start=start, end=start + span_length)
206
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/jiant/tasks/lib/wsc.py b/jiant/tasks/lib/wsc.py
--- a/jiant/tasks/lib/wsc.py
+++ b/jiant/tasks/lib/wsc.py
@@ -174,10 +174,10 @@
                     # NOTE: WSCTask.super_glue_format_preds() is dependent on this guid format.
                     guid="%s-%s" % (set_type, line["idx"]),
                     text=line["text"],
-                    span1_idx=line["span1_index"],
-                    span2_idx=line["span2_index"],
-                    span1_text=line["span1_text"],
-                    span2_text=line["span2_text"],
+                    span1_idx=line["target"]["span1_index"],
+                    span2_idx=line["target"]["span2_index"],
+                    span1_text=line["target"]["span1_text"],
+                    span2_text=line["target"]["span2_text"],
                     label=line["label"] if set_type != "test" else cls.LABELS[-1],
                 )
             )

{"golden_diff": "diff --git a/jiant/tasks/lib/wsc.py b/jiant/tasks/lib/wsc.py\n--- a/jiant/tasks/lib/wsc.py\n+++ b/jiant/tasks/lib/wsc.py\n@@ -174,10 +174,10 @@\n                     # NOTE: WSCTask.super_glue_format_preds() is dependent on this guid format.\n                     guid=\"%s-%s\" % (set_type, line[\"idx\"]),\n                     text=line[\"text\"],\n-                    span1_idx=line[\"span1_index\"],\n-                    span2_idx=line[\"span2_index\"],\n-                    span1_text=line[\"span1_text\"],\n-                    span2_text=line[\"span2_text\"],\n+                    span1_idx=line[\"target\"][\"span1_index\"],\n+                    span2_idx=line[\"target\"][\"span2_index\"],\n+                    span1_text=line[\"target\"][\"span1_text\"],\n+                    span2_text=line[\"target\"][\"span2_text\"],\n                     label=line[\"label\"] if set_type != \"test\" else cls.LABELS[-1],\n                 )\n             )\n", "issue": "wsc get_train_examples doesn't match wsc data format\nIn jiant/jiant/tasks/lib/wsc.py, `_create_examples()` dies at the statement `span1_idx=line[\"span1_index\"]` (line 177) with `KeyError: 'span1_index'` because it mismatches the structure of the JSON task data.\r\n\r\nThe statement should be `span1_idx=line[\"target\"][\"span1_index\"]` and similarly for the next 3 statements.\r\n\r\n**To Reproduce**\r\n1. Install jiant v2 from any recent commit (where wsc.py hasn't been touched since 3bd801ce)\r\n2. I doubt this matters, but running Python version 3.8 in a recent linux on a 40 core, 80 thread skylake CPU with 384 GB of RAM and a VT100/16GB GPU.\r\n3. In the Python REPL,\r\n```python\r\nfrom jiant.proj.simple import runscript as run\r\nimport jiant.scripts.download_data.runscript as downloader\r\ndownloader.download_data([\"wsc\"], \"/home/rasmussen.63/wsc-speed/tasks\")\r\nargs = run.RunConfiguration(\r\n   run_name=\"wsc-speed\",\r\n   exp_dir=\"wsc-speed\",\r\n   data_dir=\"wsc-speed/tasks\",\r\n   model_type=\"roberta-base\",\r\n   tasks=\"wsc\",\r\n   train_batch_size=16,\r\n   num_train_epochs=3\r\n)\r\nrun.run_simple(args)\r\n```\r\nWatch stderr come back to you until it stops at\r\n\r\n```\r\nTokenizing Task 'wsc' for phases 'train,val,test'\r\nWSCTask\r\n  [train]: /home/rasmussen.63/wsc-speed/tasks/data/wsc/train.jsonl\r\n  [val]: /home/rasmussen.63/wsc-speed/tasks/data/wsc/val.jsonl\r\n  [test]: /home/rasmussen.63/wsc-speed/tasks/data/wsc/test.jsonl\r\nTraceback (most recent call last):\r\n  File \"<stdin>\", line 1, in <module>\r\n  File \"/home/rasmussen.63/jiant/jiant/proj/simple/runscript.py\", line 148, in run_simple\r\n    tokenize_and_cache.main(\r\n  File \"/home/rasmussen.63/jiant/jiant/proj/main/tokenize_and_cache.py\", line 165, in main\r\n    examples=task.get_train_examples(),\r\n  File \"/home/rasmussen.63/jiant/jiant/tasks/lib/wsc.py\", line 160, in get_train_examples\r\n    return self._create_examples(lines=read_json_lines(self.train_path), set_type=\"train\")\r\n  File \"/home/rasmussen.63/jiant/jiant/tasks/lib/wsc.py\", line 177, in _create_examples\r\n    span1_idx=line[\"span1_index\"],\r\nKeyError: 'span1_index'\r\n```\r\n\r\n**Expected behavior**\r\nWSCTask should initialize an Example from the downloaded data.\r\n\r\n\n", "before_files": [{"content": "import numpy as np\nimport torch\nfrom dataclasses import dataclass\nfrom typing import List\n\nfrom jiant.tasks.core import (\n    BaseExample,\n    BaseTokenizedExample,\n    BaseDataRow,\n    BatchMixin,\n    SuperGlueMixin,\n    Task,\n    TaskTypes,\n)\nfrom jiant.tasks.lib.templates.shared import (\n    labels_to_bimap,\n    add_cls_token,\n    create_input_set_from_tokens_and_segments,\n)\nfrom jiant.tasks.utils import truncate_sequences, ExclusiveSpan\nfrom jiant.utils import retokenize\nfrom jiant.utils.python.io import read_json_lines\nfrom jiant.utils.tokenization_normalization import normalize_tokenizations\n\n\n@dataclass\nclass Example(BaseExample):\n    guid: str\n    text: str\n    span1_idx: int\n    span2_idx: int\n    span1_text: str\n    span2_text: str\n    label: str\n\n    def tokenize(self, tokenizer):\n        space_tokenization = self.text.split()\n        target_tokenization = tokenizer.tokenize(self.text)\n        normed_space_tokenization, normed_target_tokenization = normalize_tokenizations(\n            space_tokenization, target_tokenization, tokenizer\n        )\n        aligner = retokenize.TokenAligner(normed_space_tokenization, normed_target_tokenization)\n        span1_token_count = len(self.span1_text.split())\n        span2_token_count = len(self.span2_text.split())\n        target_span1 = ExclusiveSpan(\n            *aligner.project_token_span(self.span1_idx, self.span1_idx + span1_token_count)\n        )\n        target_span2 = ExclusiveSpan(\n            *aligner.project_token_span(self.span2_idx, self.span2_idx + span2_token_count)\n        )\n        return TokenizedExample(\n            guid=self.guid,\n            tokens=target_tokenization,\n            span1_span=target_span1,\n            span2_span=target_span2,\n            span1_text=self.span1_text,\n            span2_text=self.span2_text,\n            label_id=WSCTask.LABEL_TO_ID[self.label],\n        )\n\n\n@dataclass\nclass TokenizedExample(BaseTokenizedExample):\n    guid: str\n    tokens: List\n    span1_span: ExclusiveSpan\n    span2_span: ExclusiveSpan\n    span1_text: str\n    span2_text: str\n    label_id: int\n\n    def featurize(self, tokenizer, feat_spec):\n        special_tokens_count = 2  # CLS, SEP\n\n        (tokens,) = truncate_sequences(\n            tokens_ls=[self.tokens], max_length=feat_spec.max_seq_length - special_tokens_count,\n        )\n\n        unpadded_tokens = tokens + [tokenizer.sep_token]\n        unpadded_segment_ids = [feat_spec.sequence_a_segment_id] * (len(self.tokens) + 1)\n\n        unpadded_inputs = add_cls_token(\n            unpadded_tokens=unpadded_tokens,\n            unpadded_segment_ids=unpadded_segment_ids,\n            tokenizer=tokenizer,\n            feat_spec=feat_spec,\n        )\n\n        input_set = create_input_set_from_tokens_and_segments(\n            unpadded_tokens=unpadded_inputs.unpadded_tokens,\n            unpadded_segment_ids=unpadded_inputs.unpadded_segment_ids,\n            tokenizer=tokenizer,\n            feat_spec=feat_spec,\n        )\n        span1_span = ExclusiveSpan(\n            start=self.span1_span[0] + unpadded_inputs.cls_offset,\n            end=self.span1_span[1] + unpadded_inputs.cls_offset,\n        ).to_inclusive()\n        span2_span = ExclusiveSpan(\n            start=self.span2_span[0] + unpadded_inputs.cls_offset,\n            end=self.span2_span[1] + unpadded_inputs.cls_offset,\n        ).to_inclusive()\n\n        return DataRow(\n            guid=self.guid,\n            input_ids=np.array(input_set.input_ids),\n            input_mask=np.array(input_set.input_mask),\n            segment_ids=np.array(input_set.segment_ids),\n            spans=np.array([span1_span, span2_span]),\n            label_id=self.label_id,\n            tokens=unpadded_inputs.unpadded_tokens,\n            span1_text=self.span1_text,\n            span2_text=self.span2_text,\n        )\n\n\n@dataclass\nclass DataRow(BaseDataRow):\n    guid: str\n    input_ids: np.ndarray\n    input_mask: np.ndarray\n    segment_ids: np.ndarray\n    spans: np.ndarray\n    label_id: int\n    tokens: List\n    span1_text: str\n    span2_text: str\n\n    def get_tokens(self):\n        return [self.tokens]\n\n\n@dataclass\nclass Batch(BatchMixin):\n    input_ids: torch.LongTensor\n    input_mask: torch.LongTensor\n    segment_ids: torch.LongTensor\n    spans: torch.LongTensor\n    label_id: torch.LongTensor\n    tokens: List\n    span1_text: List\n    span2_text: List\n\n\nclass WSCTask(SuperGlueMixin, Task):\n    Example = Example\n    TokenizedExample = Example\n    DataRow = DataRow\n    Batch = Batch\n\n    TASK_TYPE = TaskTypes.SPAN_COMPARISON_CLASSIFICATION\n    LABELS = [False, True]\n    LABEL_TO_ID, ID_TO_LABEL = labels_to_bimap(LABELS)\n\n    @property\n    def num_spans(self):\n        return 2\n\n    def get_train_examples(self):\n        return self._create_examples(lines=read_json_lines(self.train_path), set_type=\"train\")\n\n    def get_val_examples(self):\n        return self._create_examples(lines=read_json_lines(self.val_path), set_type=\"val\")\n\n    def get_test_examples(self):\n        return self._create_examples(lines=read_json_lines(self.test_path), set_type=\"test\")\n\n    @classmethod\n    def _create_examples(cls, lines, set_type):\n        examples = []\n        for line in lines:\n            examples.append(\n                Example(\n                    # NOTE: WSCTask.super_glue_format_preds() is dependent on this guid format.\n                    guid=\"%s-%s\" % (set_type, line[\"idx\"]),\n                    text=line[\"text\"],\n                    span1_idx=line[\"span1_index\"],\n                    span2_idx=line[\"span2_index\"],\n                    span1_text=line[\"span1_text\"],\n                    span2_text=line[\"span2_text\"],\n                    label=line[\"label\"] if set_type != \"test\" else cls.LABELS[-1],\n                )\n            )\n        return examples\n\n    @classmethod\n    def super_glue_format_preds(cls, pred_dict):\n        \"\"\"Reformat this task's raw predictions to have the structure expected by SuperGLUE.\"\"\"\n        lines = []\n        for pred, guid in zip(list(pred_dict[\"preds\"]), list(pred_dict[\"guids\"])):\n            lines.append({\"idx\": int(guid.split(\"-\")[1]), \"label\": str(cls.LABELS[pred])})\n        return lines\n\n\ndef extract_char_span(full_text, span_text, space_index):\n    space_tokens = full_text.split()\n    extracted_span_text = space_tokens[space_index]\n    assert extracted_span_text.lower() in full_text.lower()\n    span_length = len(span_text)\n    if space_index == 0:\n        start = 0\n    else:\n        start = len(\" \".join(space_tokens[:space_index])) + 1\n    # exclusive span\n    return ExclusiveSpan(start=start, end=start + span_length)\n", "path": "jiant/tasks/lib/wsc.py"}], "after_files": [{"content": "import numpy as np\nimport torch\nfrom dataclasses import dataclass\nfrom typing import List\n\nfrom jiant.tasks.core import (\n    BaseExample,\n    BaseTokenizedExample,\n    BaseDataRow,\n    BatchMixin,\n    SuperGlueMixin,\n    Task,\n    TaskTypes,\n)\nfrom jiant.tasks.lib.templates.shared import (\n    labels_to_bimap,\n    add_cls_token,\n    create_input_set_from_tokens_and_segments,\n)\nfrom jiant.tasks.utils import truncate_sequences, ExclusiveSpan\nfrom jiant.utils import retokenize\nfrom jiant.utils.python.io import read_json_lines\nfrom jiant.utils.tokenization_normalization import normalize_tokenizations\n\n\n@dataclass\nclass Example(BaseExample):\n    guid: str\n    text: str\n    span1_idx: int\n    span2_idx: int\n    span1_text: str\n    span2_text: str\n    label: str\n\n    def tokenize(self, tokenizer):\n        space_tokenization = self.text.split()\n        target_tokenization = tokenizer.tokenize(self.text)\n        normed_space_tokenization, normed_target_tokenization = normalize_tokenizations(\n            space_tokenization, target_tokenization, tokenizer\n        )\n        aligner = retokenize.TokenAligner(normed_space_tokenization, normed_target_tokenization)\n        span1_token_count = len(self.span1_text.split())\n        span2_token_count = len(self.span2_text.split())\n        target_span1 = ExclusiveSpan(\n            *aligner.project_token_span(self.span1_idx, self.span1_idx + span1_token_count)\n        )\n        target_span2 = ExclusiveSpan(\n            *aligner.project_token_span(self.span2_idx, self.span2_idx + span2_token_count)\n        )\n        return TokenizedExample(\n            guid=self.guid,\n            tokens=target_tokenization,\n            span1_span=target_span1,\n            span2_span=target_span2,\n            span1_text=self.span1_text,\n            span2_text=self.span2_text,\n            label_id=WSCTask.LABEL_TO_ID[self.label],\n        )\n\n\n@dataclass\nclass TokenizedExample(BaseTokenizedExample):\n    guid: str\n    tokens: List\n    span1_span: ExclusiveSpan\n    span2_span: ExclusiveSpan\n    span1_text: str\n    span2_text: str\n    label_id: int\n\n    def featurize(self, tokenizer, feat_spec):\n        special_tokens_count = 2  # CLS, SEP\n\n        (tokens,) = truncate_sequences(\n            tokens_ls=[self.tokens], max_length=feat_spec.max_seq_length - special_tokens_count,\n        )\n\n        unpadded_tokens = tokens + [tokenizer.sep_token]\n        unpadded_segment_ids = [feat_spec.sequence_a_segment_id] * (len(self.tokens) + 1)\n\n        unpadded_inputs = add_cls_token(\n            unpadded_tokens=unpadded_tokens,\n            unpadded_segment_ids=unpadded_segment_ids,\n            tokenizer=tokenizer,\n            feat_spec=feat_spec,\n        )\n\n        input_set = create_input_set_from_tokens_and_segments(\n            unpadded_tokens=unpadded_inputs.unpadded_tokens,\n            unpadded_segment_ids=unpadded_inputs.unpadded_segment_ids,\n            tokenizer=tokenizer,\n            feat_spec=feat_spec,\n        )\n        span1_span = ExclusiveSpan(\n            start=self.span1_span[0] + unpadded_inputs.cls_offset,\n            end=self.span1_span[1] + unpadded_inputs.cls_offset,\n        ).to_inclusive()\n        span2_span = ExclusiveSpan(\n            start=self.span2_span[0] + unpadded_inputs.cls_offset,\n            end=self.span2_span[1] + unpadded_inputs.cls_offset,\n        ).to_inclusive()\n\n        return DataRow(\n            guid=self.guid,\n            input_ids=np.array(input_set.input_ids),\n            input_mask=np.array(input_set.input_mask),\n            segment_ids=np.array(input_set.segment_ids),\n            spans=np.array([span1_span, span2_span]),\n            label_id=self.label_id,\n            tokens=unpadded_inputs.unpadded_tokens,\n            span1_text=self.span1_text,\n            span2_text=self.span2_text,\n        )\n\n\n@dataclass\nclass DataRow(BaseDataRow):\n    guid: str\n    input_ids: np.ndarray\n    input_mask: np.ndarray\n    segment_ids: np.ndarray\n    spans: np.ndarray\n    label_id: int\n    tokens: List\n    span1_text: str\n    span2_text: str\n\n    def get_tokens(self):\n        return [self.tokens]\n\n\n@dataclass\nclass Batch(BatchMixin):\n    input_ids: torch.LongTensor\n    input_mask: torch.LongTensor\n    segment_ids: torch.LongTensor\n    spans: torch.LongTensor\n    label_id: torch.LongTensor\n    tokens: List\n    span1_text: List\n    span2_text: List\n\n\nclass WSCTask(SuperGlueMixin, Task):\n    Example = Example\n    TokenizedExample = Example\n    DataRow = DataRow\n    Batch = Batch\n\n    TASK_TYPE = TaskTypes.SPAN_COMPARISON_CLASSIFICATION\n    LABELS = [False, True]\n    LABEL_TO_ID, ID_TO_LABEL = labels_to_bimap(LABELS)\n\n    @property\n    def num_spans(self):\n        return 2\n\n    def get_train_examples(self):\n        return self._create_examples(lines=read_json_lines(self.train_path), set_type=\"train\")\n\n    def get_val_examples(self):\n        return self._create_examples(lines=read_json_lines(self.val_path), set_type=\"val\")\n\n    def get_test_examples(self):\n        return self._create_examples(lines=read_json_lines(self.test_path), set_type=\"test\")\n\n    @classmethod\n    def _create_examples(cls, lines, set_type):\n        examples = []\n        for line in lines:\n            examples.append(\n                Example(\n                    # NOTE: WSCTask.super_glue_format_preds() is dependent on this guid format.\n                    guid=\"%s-%s\" % (set_type, line[\"idx\"]),\n                    text=line[\"text\"],\n                    span1_idx=line[\"target\"][\"span1_index\"],\n                    span2_idx=line[\"target\"][\"span2_index\"],\n                    span1_text=line[\"target\"][\"span1_text\"],\n                    span2_text=line[\"target\"][\"span2_text\"],\n                    label=line[\"label\"] if set_type != \"test\" else cls.LABELS[-1],\n                )\n            )\n        return examples\n\n    @classmethod\n    def super_glue_format_preds(cls, pred_dict):\n        \"\"\"Reformat this task's raw predictions to have the structure expected by SuperGLUE.\"\"\"\n        lines = []\n        for pred, guid in zip(list(pred_dict[\"preds\"]), list(pred_dict[\"guids\"])):\n            lines.append({\"idx\": int(guid.split(\"-\")[1]), \"label\": str(cls.LABELS[pred])})\n        return lines\n\n\ndef extract_char_span(full_text, span_text, space_index):\n    space_tokens = full_text.split()\n    extracted_span_text = space_tokens[space_index]\n    assert extracted_span_text.lower() in full_text.lower()\n    span_length = len(span_text)\n    if space_index == 0:\n        start = 0\n    else:\n        start = len(\" \".join(space_tokens[:space_index])) + 1\n    # exclusive span\n    return ExclusiveSpan(start=start, end=start + span_length)\n", "path": "jiant/tasks/lib/wsc.py"}]}