We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
LaTeX processing is not being done on ingestion
This came up in #628, and I think it appeared for the first time for EMNLP 2019 because we pushed some simplifying changes to START that turned off LaTeX on their end. 

Is it because `normalize_anth.py` is not being run with the `-t` option?

The problem is that currently, `normalize_anth.py` cannot be rerun with the `-t` option; all kinds of errors come up. It ought to be fixable but might not be an easy fix.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `bin/normalize_anth.py`
Content:
```
1 #!/usr/bin/env python3
2 # -*- coding: utf-8 -*-
3 #
4 # Copyright 2019 David Wei Chiang <dchiang@nd.edu>
5 #
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at
9 #
10 #     http://www.apache.org/licenses/LICENSE-2.0
11 #
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17 
18 """Try to convert ACL Anthology XML format to a standard form, in
19 which:
20 - All single and double quotes are curly
21 - Some characters (e.g., fullwidth chars) are mapped to standard equivalents
22 - Combining accents and characters are composed into single characters
23 - Letters that are capital "by nature" are protected with <fixed-case>
24 - With the `-t` option:
25   - Outside of formulas, no LaTeX is used; only Unicode
26   - Formulas are tagged as <tex-math> and use LaTeX
27 
28 Usage: python3 normalize_anth.py [-t] <infile> <outfile>
29 
30 Bugs: 
31 
32 - Doesn't preserve line breaks and indentation within text fields.
33 """
34 
35 import lxml.etree as etree
36 import re
37 import difflib
38 import logging
39 import unicodedata
40 import html
41 from latex_to_unicode import latex_to_xml
42 from fixedcase.protect import protect
43 
44 location = ""
45 logging.basicConfig(format='%(levelname)s:%(location)s %(message)s', level=logging.INFO)
46 def filter(r):
47     r.location = location
48     return True
49 logging.getLogger().addFilter(filter)
50 
51 def replace_node(old, new):
52     save_tail = old.tail
53     old.clear()
54     old.tag = new.tag
55     old.attrib.update(new.attrib)
56     old.text = new.text
57     old.extend(new)
58     old.tail = save_tail
59 
60 def maptext(node, f):
61     if node.tag in ['tex-math', 'url']:
62         return
63     if node.text is not None:
64         node.text = f(node.text)
65     for child in node:
66         maptext(child, f)
67         if child.tail is not None:
68             child.tail = f(child.tail)
69 
70 def curly_quotes(s):
71     # Two single quotes after a word: LaTeX for a right curly quote.
72     s = re.sub(r'(\w[^\s"]*)\'\'', r'\1”', s)
73     # Two single quotes (or backticks) before a word: LaTeX for a left curly quote.
74     s = re.sub(r'(``|\'\')(\w)', r'“\2', s)
75 
76     # Straight double quote: If preceded by a word (possibly with
77     # intervening punctuation), it's a right quote.
78     s = re.sub(r'(\w[^\s"]*)"', r'\1”', s)
79     # Else, if followed by a word, it's a left quote
80     s = re.sub(r'"(\w)', r'“\1', s)
81     if '"' in s: logging.warning(f"couldn't convert straight double quote in [{s}]")
82 
83     # Straight single quote
84     # Exceptions for words that start with apostrophe
85     s = re.sub(r"'(em|round|n|tis|twas|til|cause|scuse|\d0)\b", r'’\1', s, flags=re.IGNORECASE)
86     # Otherwise, treat the same as straight double quote
87     s = re.sub(r"(\w[^\s']*)'", r'\1’', s)
88     s = re.sub(r"'(\w)", r'‘\1', s)
89     if "'" in s: logging.warning(f"couldn't convert straight single quote in [{s}]")
90 
91     return s
92 
93 def clean_unicode(s):
94     s = s.replace('\u00ad', '') # soft hyphen
95     s = s.replace('\u2010', '-') # hyphen
96 
97     # Some sources encode an i with an accent above using dotless i,
98     # which must be converted to normal i
99     s = list(s)
100     for i in range(len(s)-1):
101         # bug: we should only be looking for accents above, not
102         # below
103         if s[i] == 'ı' and unicodedata.category(s[i+1]) == 'Mn':
104             s[i] = 'i'
105     s = ''.join(s)
106 
107     # Selectively apply compatibility decomposition.
108     # This converts, e.g., ﬁ to fi and ： to :, but not ² to 2.
109     # Unsure: … to ...
110     # More classes could be added here.
111     def decompose(c):
112         d = unicodedata.decomposition(c)
113         if d and d.split(None, 1)[0] in ['<compat>', '<wide>', '<narrow>', '<noBreak>']:
114             return unicodedata.normalize('NFKD', c)
115         else:
116             return c
117     s = ''.join(map(decompose, s))
118 
119     # Convert combining characters when possible
120     s = unicodedata.normalize('NFC', s)
121 
122     return s
123 
124 def process(oldnode, informat):
125     if oldnode.tag in ['url', 'href', 'mrf', 'doi', 'bibtype', 'bibkey',
126                        'revision', 'erratum', 'attachment', 'paper',
127                        'presentation', 'dataset', 'software', 'video']:
128         return
129     elif oldnode.tag in ['author', 'editor']:
130         for oldchild in oldnode:
131             process(oldchild, informat=informat)
132     else:
133         if informat == "latex":
134             if len(oldnode) > 0:
135                 logging.error("field has child elements {}".format(', '.join(child.tag for child in oldnode)))
136             oldtext = ''.join(oldnode.itertext())
137             newnode = latex_to_xml(oldtext, trivial_math=True, fixed_case=True)
138             newnode.tag = oldnode.tag
139             newnode.attrib.update(oldnode.attrib)
140             replace_node(oldnode, newnode)
141             
142         maptext(oldnode, html.unescape)
143         maptext(oldnode, curly_quotes)
144         maptext(oldnode, clean_unicode)
145         if oldnode.tag in ['title', 'booktitle']:
146             protect(oldnode)
147 
148 if __name__ == "__main__":
149     import sys
150     import argparse
151     ap = argparse.ArgumentParser(description='Convert Anthology XML to standard format.')
152     ap.add_argument('infile', help="XML file to read")
153     ap.add_argument('outfile', help="XML file to write")
154     ap.add_argument('-t', '--latex', action="store_true", help="Assume input fields are in LaTeX (not idempotent")
155     args = ap.parse_args()
156 
157     if args.latex:
158         informat = "latex"
159     else:
160         informat = "xml"
161 
162     tree = etree.parse(args.infile)
163     root = tree.getroot()
164     if not root.tail:
165         # lxml drops trailing newline
166         root.tail = "\n"
167     for paper in root.findall('.//paper'):
168         papernum = "{} vol {} paper {}".format(root.attrib['id'],
169                                                paper.getparent().attrib['id'],
170                                                paper.attrib['id'])
171         for oldnode in paper:
172             location = "{}:{}".format(papernum, oldnode.tag)
173             process(oldnode, informat=informat)
174                 
175     tree.write(args.outfile, encoding="UTF-8", xml_declaration=True)
176
```

Path: `bin/ingest.py`
Content:
```
1 #!/usr/bin/env python3
2 # -*- coding: utf-8 -*-
3 #
4 # Copyright 2019 Matt Post <post@cs.jhu.edu>
5 #
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at
9 #
10 #     http://www.apache.org/licenses/LICENSE-2.0
11 #
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17 
18 """
19 Ingests data into the Anthology. It takes an XML file and:
20 
21 - converts to nested format if not already done
22 - applies normalization (fixed-case protection)
23 - runs sanity checks (e.g., % of authors in input data that are new)
24 """
25 
26 import argparse
27 import os
28 import re
29 import sys
30 
31 import lxml.etree as etree
32 
33 from datetime import datetime
34 
35 from normalize_anth import process
36 from anthology.utils import make_nested, make_simple_element, build_anthology_id, indent
37 from anthology.index import AnthologyIndex
38 from anthology.people import PersonName
39 
40 from itertools import chain
41 
42 
43 if __name__ == '__main__':
44     now = datetime.now()
45     today = f'{now.year}-{now.month:02d}-{now.day:02d}'
46 
47     parser = argparse.ArgumentParser()
48     parser.add_argument('infile')
49     parser.add_argument('--ingest-date', '-d', type=str, default=today, help='Ingestion date as YYYY-MM-DD. Default: %(default)s.')
50     parser.add_argument('--append', '-a', action='store_true', help='Append to existing volume instead of quitting.')
51     args = parser.parse_args()
52 
53     people = AnthologyIndex(None, srcdir=os.path.join(os.path.dirname(sys.argv[0]), '..', 'data'))
54 
55     pdf_directory = os.path.dirname(args.infile)
56     tree_being_added = etree.parse(args.infile)
57 
58     # Ensure nested format
59     root_being_added = make_nested(tree_being_added.getroot(), pdf_path=os.path.dirname(args.infile))
60     collection_id = root_being_added.attrib['id']
61 
62     # Ensure names are properly identified
63     ambiguous = {}
64     for paper in root_being_added.findall('.//paper'):
65         anth_id = build_anthology_id(collection_id,
66                                      paper.getparent().attrib['id'],
67                                      paper.attrib['id'])
68 
69         for node in chain(paper.findall('author'), paper.findall('editor')):
70             name = PersonName.from_element(node)
71             ids = people.get_ids(name)
72             if len(ids) > 1:
73                 print(f'WARNING ({anth_id}): ambiguous author {name}, defaulting to first of {ids}')
74                 ambiguous[anth_id] = (name, ids)
75 
76                 node.attrib['id'] = ids[0]
77 
78         # Ensure PDF exists. PDF should be in the same directory as the XML file being ingested.
79         pdf_path = os.path.join(pdf_directory, f'{anth_id}.pdf')
80         if not os.path.exists(pdf_path):
81             print(f'FATAL: cannot file PDF {pdf_path}!')
82             sys.exit(1)
83 
84     # Normalize
85     for paper in root_being_added.findall('.//paper'):
86         for oldnode in paper:
87             process(oldnode, informat='xml')
88 
89     # Ingest each volume.
90     # First, find the XML file.
91     collection_file = os.path.join(os.path.dirname(sys.argv[0]), '..', 'data', 'xml', f'{collection_id}.xml')
92 
93     if os.path.exists(collection_file):
94         existing_tree = etree.parse(collection_file)
95     else:
96         existing_tree = etree.ElementTree(make_simple_element('collection', attrib={'id': collection_id}))
97 
98     # Insert each volume
99     for i, new_volume in enumerate(root_being_added.findall('volume')):
100         new_volume_id = int(new_volume.attrib['id'])
101         existing_volume = existing_tree.getroot().find(f"./volume[@id='{new_volume_id}']")
102         if existing_volume is None:
103             new_volume.attrib['ingest-date'] = args.ingest_date
104 
105             # Find the insertion point among the other volumes
106             insertion_point = 0
107             for i, volume in enumerate(existing_tree.getroot()):
108                 if new_volume_id < int(volume.attrib['id']):
109                     break
110                 insertion_point = i + 1
111             print(f"Inserting volume {new_volume_id} at collection position {insertion_point}")
112             existing_tree.getroot().insert(insertion_point, new_volume)
113         else:
114             # Append to existing volume (useful for TACL, which has a single volume each year) if requested
115             if args.append:
116                 for paper in new_volume.findall('./paper'):
117                     print(f'Appending {paper.attrib["id"]}')
118                     paper.attrib['ingest-date'] = args.ingest_date
119                     existing_volume.append(paper)
120             else:
121                 print(f'Volume {new_volume_id} has already been inserted into {collection_file}.')
122                 print(f'You can append to this volume by passing `--append` (or `-a`) to this script.')
123                 print(f'Quitting, since you didn\'t.')
124                 sys.exit(1)
125 
126     indent(existing_tree.getroot())
127     existing_tree.write(collection_file, encoding='UTF-8', xml_declaration=True, with_tail=True)
128
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/bin/ingest.py b/bin/ingest.py
--- a/bin/ingest.py
+++ b/bin/ingest.py
@@ -32,7 +32,7 @@
 
 from datetime import datetime
 
-from normalize_anth import process
+from normalize_anth import normalize
 from anthology.utils import make_nested, make_simple_element, build_anthology_id, indent
 from anthology.index import AnthologyIndex
 from anthology.people import PersonName
@@ -84,7 +84,7 @@
     # Normalize
     for paper in root_being_added.findall('.//paper'):
         for oldnode in paper:
-            process(oldnode, informat='xml')
+            normalize(oldnode, informat='latex')
 
     # Ingest each volume.
     # First, find the XML file.
diff --git a/bin/normalize_anth.py b/bin/normalize_anth.py
--- a/bin/normalize_anth.py
+++ b/bin/normalize_anth.py
@@ -121,14 +121,24 @@
 
     return s
 
-def process(oldnode, informat):
+
+def normalize(oldnode, informat):
+    """
+    Receives an XML 'paper' node and normalizes many of its fields, including:
+    - Unescaping HTML
+    - Normalizing quotes and other punctuation
+    - Mapping many characters to unicode
+    In addition, if the 'informat' is "latex", it will convert many LaTeX characters
+    to unicode equivalents. Note that these latter LaTeX operations are not idempotent.
+    """
+
     if oldnode.tag in ['url', 'href', 'mrf', 'doi', 'bibtype', 'bibkey',
                        'revision', 'erratum', 'attachment', 'paper',
                        'presentation', 'dataset', 'software', 'video']:
         return
     elif oldnode.tag in ['author', 'editor']:
         for oldchild in oldnode:
-            process(oldchild, informat=informat)
+            normalize(oldchild, informat=informat)
     else:
         if informat == "latex":
             if len(oldnode) > 0:
@@ -138,7 +148,7 @@
             newnode.tag = oldnode.tag
             newnode.attrib.update(oldnode.attrib)
             replace_node(oldnode, newnode)
-            
+
         maptext(oldnode, html.unescape)
         maptext(oldnode, curly_quotes)
         maptext(oldnode, clean_unicode)
@@ -170,6 +180,6 @@
                                                paper.attrib['id'])
         for oldnode in paper:
             location = "{}:{}".format(papernum, oldnode.tag)
-            process(oldnode, informat=informat)
-                
+            normalize(oldnode, informat=informat)
+
     tree.write(args.outfile, encoding="UTF-8", xml_declaration=True)

{"golden_diff": "diff --git a/bin/ingest.py b/bin/ingest.py\n--- a/bin/ingest.py\n+++ b/bin/ingest.py\n@@ -32,7 +32,7 @@\n \n from datetime import datetime\n \n-from normalize_anth import process\n+from normalize_anth import normalize\n from anthology.utils import make_nested, make_simple_element, build_anthology_id, indent\n from anthology.index import AnthologyIndex\n from anthology.people import PersonName\n@@ -84,7 +84,7 @@\n     # Normalize\n     for paper in root_being_added.findall('.//paper'):\n         for oldnode in paper:\n-            process(oldnode, informat='xml')\n+            normalize(oldnode, informat='latex')\n \n     # Ingest each volume.\n     # First, find the XML file.\ndiff --git a/bin/normalize_anth.py b/bin/normalize_anth.py\n--- a/bin/normalize_anth.py\n+++ b/bin/normalize_anth.py\n@@ -121,14 +121,24 @@\n \n     return s\n \n-def process(oldnode, informat):\n+\n+def normalize(oldnode, informat):\n+    \"\"\"\n+    Receives an XML 'paper' node and normalizes many of its fields, including:\n+    - Unescaping HTML\n+    - Normalizing quotes and other punctuation\n+    - Mapping many characters to unicode\n+    In addition, if the 'informat' is \"latex\", it will convert many LaTeX characters\n+    to unicode equivalents. Note that these latter LaTeX operations are not idempotent.\n+    \"\"\"\n+\n     if oldnode.tag in ['url', 'href', 'mrf', 'doi', 'bibtype', 'bibkey',\n                        'revision', 'erratum', 'attachment', 'paper',\n                        'presentation', 'dataset', 'software', 'video']:\n         return\n     elif oldnode.tag in ['author', 'editor']:\n         for oldchild in oldnode:\n-            process(oldchild, informat=informat)\n+            normalize(oldchild, informat=informat)\n     else:\n         if informat == \"latex\":\n             if len(oldnode) > 0:\n@@ -138,7 +148,7 @@\n             newnode.tag = oldnode.tag\n             newnode.attrib.update(oldnode.attrib)\n             replace_node(oldnode, newnode)\n-            \n+\n         maptext(oldnode, html.unescape)\n         maptext(oldnode, curly_quotes)\n         maptext(oldnode, clean_unicode)\n@@ -170,6 +180,6 @@\n                                                paper.attrib['id'])\n         for oldnode in paper:\n             location = \"{}:{}\".format(papernum, oldnode.tag)\n-            process(oldnode, informat=informat)\n-                \n+            normalize(oldnode, informat=informat)\n+\n     tree.write(args.outfile, encoding=\"UTF-8\", xml_declaration=True)\n", "issue": "LaTeX processing is not being done on ingestion\nThis came up in #628, and I think it appeared for the first time for EMNLP 2019 because we pushed some simplifying changes to START that turned off LaTeX on their end. \r\n\r\nIs it because `normalize_anth.py` is not being run with the `-t` option?\r\n\r\nThe problem is that currently, `normalize_anth.py` cannot be rerun with the `-t` option; all kinds of errors come up. It ought to be fixable but might not be an easy fix.\n", "before_files": [{"content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright 2019 David Wei Chiang <dchiang@nd.edu>\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"Try to convert ACL Anthology XML format to a standard form, in\nwhich:\n- All single and double quotes are curly\n- Some characters (e.g., fullwidth chars) are mapped to standard equivalents\n- Combining accents and characters are composed into single characters\n- Letters that are capital \"by nature\" are protected with <fixed-case>\n- With the `-t` option:\n  - Outside of formulas, no LaTeX is used; only Unicode\n  - Formulas are tagged as <tex-math> and use LaTeX\n\nUsage: python3 normalize_anth.py [-t] <infile> <outfile>\n\nBugs: \n\n- Doesn't preserve line breaks and indentation within text fields.\n\"\"\"\n\nimport lxml.etree as etree\nimport re\nimport difflib\nimport logging\nimport unicodedata\nimport html\nfrom latex_to_unicode import latex_to_xml\nfrom fixedcase.protect import protect\n\nlocation = \"\"\nlogging.basicConfig(format='%(levelname)s:%(location)s %(message)s', level=logging.INFO)\ndef filter(r):\n    r.location = location\n    return True\nlogging.getLogger().addFilter(filter)\n\ndef replace_node(old, new):\n    save_tail = old.tail\n    old.clear()\n    old.tag = new.tag\n    old.attrib.update(new.attrib)\n    old.text = new.text\n    old.extend(new)\n    old.tail = save_tail\n\ndef maptext(node, f):\n    if node.tag in ['tex-math', 'url']:\n        return\n    if node.text is not None:\n        node.text = f(node.text)\n    for child in node:\n        maptext(child, f)\n        if child.tail is not None:\n            child.tail = f(child.tail)\n\ndef curly_quotes(s):\n    # Two single quotes after a word: LaTeX for a right curly quote.\n    s = re.sub(r'(\\w[^\\s\"]*)\\'\\'', r'\\1\u201d', s)\n    # Two single quotes (or backticks) before a word: LaTeX for a left curly quote.\n    s = re.sub(r'(``|\\'\\')(\\w)', r'\u201c\\2', s)\n\n    # Straight double quote: If preceded by a word (possibly with\n    # intervening punctuation), it's a right quote.\n    s = re.sub(r'(\\w[^\\s\"]*)\"', r'\\1\u201d', s)\n    # Else, if followed by a word, it's a left quote\n    s = re.sub(r'\"(\\w)', r'\u201c\\1', s)\n    if '\"' in s: logging.warning(f\"couldn't convert straight double quote in [{s}]\")\n\n    # Straight single quote\n    # Exceptions for words that start with apostrophe\n    s = re.sub(r\"'(em|round|n|tis|twas|til|cause|scuse|\\d0)\\b\", r'\u2019\\1', s, flags=re.IGNORECASE)\n    # Otherwise, treat the same as straight double quote\n    s = re.sub(r\"(\\w[^\\s']*)'\", r'\\1\u2019', s)\n    s = re.sub(r\"'(\\w)\", r'\u2018\\1', s)\n    if \"'\" in s: logging.warning(f\"couldn't convert straight single quote in [{s}]\")\n\n    return s\n\ndef clean_unicode(s):\n    s = s.replace('\\u00ad', '') # soft hyphen\n    s = s.replace('\\u2010', '-') # hyphen\n\n    # Some sources encode an i with an accent above using dotless i,\n    # which must be converted to normal i\n    s = list(s)\n    for i in range(len(s)-1):\n        # bug: we should only be looking for accents above, not\n        # below\n        if s[i] == '\u0131' and unicodedata.category(s[i+1]) == 'Mn':\n            s[i] = 'i'\n    s = ''.join(s)\n\n    # Selectively apply compatibility decomposition.\n    # This converts, e.g., \ufb01 to fi and \uff1a to :, but not \u00b2 to 2.\n    # Unsure: \u2026 to ...\n    # More classes could be added here.\n    def decompose(c):\n        d = unicodedata.decomposition(c)\n        if d and d.split(None, 1)[0] in ['<compat>', '<wide>', '<narrow>', '<noBreak>']:\n            return unicodedata.normalize('NFKD', c)\n        else:\n            return c\n    s = ''.join(map(decompose, s))\n\n    # Convert combining characters when possible\n    s = unicodedata.normalize('NFC', s)\n\n    return s\n\ndef process(oldnode, informat):\n    if oldnode.tag in ['url', 'href', 'mrf', 'doi', 'bibtype', 'bibkey',\n                       'revision', 'erratum', 'attachment', 'paper',\n                       'presentation', 'dataset', 'software', 'video']:\n        return\n    elif oldnode.tag in ['author', 'editor']:\n        for oldchild in oldnode:\n            process(oldchild, informat=informat)\n    else:\n        if informat == \"latex\":\n            if len(oldnode) > 0:\n                logging.error(\"field has child elements {}\".format(', '.join(child.tag for child in oldnode)))\n            oldtext = ''.join(oldnode.itertext())\n            newnode = latex_to_xml(oldtext, trivial_math=True, fixed_case=True)\n            newnode.tag = oldnode.tag\n            newnode.attrib.update(oldnode.attrib)\n            replace_node(oldnode, newnode)\n            \n        maptext(oldnode, html.unescape)\n        maptext(oldnode, curly_quotes)\n        maptext(oldnode, clean_unicode)\n        if oldnode.tag in ['title', 'booktitle']:\n            protect(oldnode)\n\nif __name__ == \"__main__\":\n    import sys\n    import argparse\n    ap = argparse.ArgumentParser(description='Convert Anthology XML to standard format.')\n    ap.add_argument('infile', help=\"XML file to read\")\n    ap.add_argument('outfile', help=\"XML file to write\")\n    ap.add_argument('-t', '--latex', action=\"store_true\", help=\"Assume input fields are in LaTeX (not idempotent\")\n    args = ap.parse_args()\n\n    if args.latex:\n        informat = \"latex\"\n    else:\n        informat = \"xml\"\n\n    tree = etree.parse(args.infile)\n    root = tree.getroot()\n    if not root.tail:\n        # lxml drops trailing newline\n        root.tail = \"\\n\"\n    for paper in root.findall('.//paper'):\n        papernum = \"{} vol {} paper {}\".format(root.attrib['id'],\n                                               paper.getparent().attrib['id'],\n                                               paper.attrib['id'])\n        for oldnode in paper:\n            location = \"{}:{}\".format(papernum, oldnode.tag)\n            process(oldnode, informat=informat)\n                \n    tree.write(args.outfile, encoding=\"UTF-8\", xml_declaration=True)\n", "path": "bin/normalize_anth.py"}, {"content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright 2019 Matt Post <post@cs.jhu.edu>\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nIngests data into the Anthology. It takes an XML file and:\n\n- converts to nested format if not already done\n- applies normalization (fixed-case protection)\n- runs sanity checks (e.g., % of authors in input data that are new)\n\"\"\"\n\nimport argparse\nimport os\nimport re\nimport sys\n\nimport lxml.etree as etree\n\nfrom datetime import datetime\n\nfrom normalize_anth import process\nfrom anthology.utils import make_nested, make_simple_element, build_anthology_id, indent\nfrom anthology.index import AnthologyIndex\nfrom anthology.people import PersonName\n\nfrom itertools import chain\n\n\nif __name__ == '__main__':\n    now = datetime.now()\n    today = f'{now.year}-{now.month:02d}-{now.day:02d}'\n\n    parser = argparse.ArgumentParser()\n    parser.add_argument('infile')\n    parser.add_argument('--ingest-date', '-d', type=str, default=today, help='Ingestion date as YYYY-MM-DD. Default: %(default)s.')\n    parser.add_argument('--append', '-a', action='store_true', help='Append to existing volume instead of quitting.')\n    args = parser.parse_args()\n\n    people = AnthologyIndex(None, srcdir=os.path.join(os.path.dirname(sys.argv[0]), '..', 'data'))\n\n    pdf_directory = os.path.dirname(args.infile)\n    tree_being_added = etree.parse(args.infile)\n\n    # Ensure nested format\n    root_being_added = make_nested(tree_being_added.getroot(), pdf_path=os.path.dirname(args.infile))\n    collection_id = root_being_added.attrib['id']\n\n    # Ensure names are properly identified\n    ambiguous = {}\n    for paper in root_being_added.findall('.//paper'):\n        anth_id = build_anthology_id(collection_id,\n                                     paper.getparent().attrib['id'],\n                                     paper.attrib['id'])\n\n        for node in chain(paper.findall('author'), paper.findall('editor')):\n            name = PersonName.from_element(node)\n            ids = people.get_ids(name)\n            if len(ids) > 1:\n                print(f'WARNING ({anth_id}): ambiguous author {name}, defaulting to first of {ids}')\n                ambiguous[anth_id] = (name, ids)\n\n                node.attrib['id'] = ids[0]\n\n        # Ensure PDF exists. PDF should be in the same directory as the XML file being ingested.\n        pdf_path = os.path.join(pdf_directory, f'{anth_id}.pdf')\n        if not os.path.exists(pdf_path):\n            print(f'FATAL: cannot file PDF {pdf_path}!')\n            sys.exit(1)\n\n    # Normalize\n    for paper in root_being_added.findall('.//paper'):\n        for oldnode in paper:\n            process(oldnode, informat='xml')\n\n    # Ingest each volume.\n    # First, find the XML file.\n    collection_file = os.path.join(os.path.dirname(sys.argv[0]), '..', 'data', 'xml', f'{collection_id}.xml')\n\n    if os.path.exists(collection_file):\n        existing_tree = etree.parse(collection_file)\n    else:\n        existing_tree = etree.ElementTree(make_simple_element('collection', attrib={'id': collection_id}))\n\n    # Insert each volume\n    for i, new_volume in enumerate(root_being_added.findall('volume')):\n        new_volume_id = int(new_volume.attrib['id'])\n        existing_volume = existing_tree.getroot().find(f\"./volume[@id='{new_volume_id}']\")\n        if existing_volume is None:\n            new_volume.attrib['ingest-date'] = args.ingest_date\n\n            # Find the insertion point among the other volumes\n            insertion_point = 0\n            for i, volume in enumerate(existing_tree.getroot()):\n                if new_volume_id < int(volume.attrib['id']):\n                    break\n                insertion_point = i + 1\n            print(f\"Inserting volume {new_volume_id} at collection position {insertion_point}\")\n            existing_tree.getroot().insert(insertion_point, new_volume)\n        else:\n            # Append to existing volume (useful for TACL, which has a single volume each year) if requested\n            if args.append:\n                for paper in new_volume.findall('./paper'):\n                    print(f'Appending {paper.attrib[\"id\"]}')\n                    paper.attrib['ingest-date'] = args.ingest_date\n                    existing_volume.append(paper)\n            else:\n                print(f'Volume {new_volume_id} has already been inserted into {collection_file}.')\n                print(f'You can append to this volume by passing `--append` (or `-a`) to this script.')\n                print(f'Quitting, since you didn\\'t.')\n                sys.exit(1)\n\n    indent(existing_tree.getroot())\n    existing_tree.write(collection_file, encoding='UTF-8', xml_declaration=True, with_tail=True)\n", "path": "bin/ingest.py"}], "after_files": [{"content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright 2019 David Wei Chiang <dchiang@nd.edu>\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"Try to convert ACL Anthology XML format to a standard form, in\nwhich:\n- All single and double quotes are curly\n- Some characters (e.g., fullwidth chars) are mapped to standard equivalents\n- Combining accents and characters are composed into single characters\n- Letters that are capital \"by nature\" are protected with <fixed-case>\n- With the `-t` option:\n  - Outside of formulas, no LaTeX is used; only Unicode\n  - Formulas are tagged as <tex-math> and use LaTeX\n\nUsage: python3 normalize_anth.py [-t] <infile> <outfile>\n\nBugs: \n\n- Doesn't preserve line breaks and indentation within text fields.\n\"\"\"\n\nimport lxml.etree as etree\nimport re\nimport difflib\nimport logging\nimport unicodedata\nimport html\nfrom latex_to_unicode import latex_to_xml\nfrom fixedcase.protect import protect\n\nlocation = \"\"\nlogging.basicConfig(format='%(levelname)s:%(location)s %(message)s', level=logging.INFO)\ndef filter(r):\n    r.location = location\n    return True\nlogging.getLogger().addFilter(filter)\n\ndef replace_node(old, new):\n    save_tail = old.tail\n    old.clear()\n    old.tag = new.tag\n    old.attrib.update(new.attrib)\n    old.text = new.text\n    old.extend(new)\n    old.tail = save_tail\n\ndef maptext(node, f):\n    if node.tag in ['tex-math', 'url']:\n        return\n    if node.text is not None:\n        node.text = f(node.text)\n    for child in node:\n        maptext(child, f)\n        if child.tail is not None:\n            child.tail = f(child.tail)\n\ndef curly_quotes(s):\n    # Two single quotes after a word: LaTeX for a right curly quote.\n    s = re.sub(r'(\\w[^\\s\"]*)\\'\\'', r'\\1\u201d', s)\n    # Two single quotes (or backticks) before a word: LaTeX for a left curly quote.\n    s = re.sub(r'(``|\\'\\')(\\w)', r'\u201c\\2', s)\n\n    # Straight double quote: If preceded by a word (possibly with\n    # intervening punctuation), it's a right quote.\n    s = re.sub(r'(\\w[^\\s\"]*)\"', r'\\1\u201d', s)\n    # Else, if followed by a word, it's a left quote\n    s = re.sub(r'\"(\\w)', r'\u201c\\1', s)\n    if '\"' in s: logging.warning(f\"couldn't convert straight double quote in [{s}]\")\n\n    # Straight single quote\n    # Exceptions for words that start with apostrophe\n    s = re.sub(r\"'(em|round|n|tis|twas|til|cause|scuse|\\d0)\\b\", r'\u2019\\1', s, flags=re.IGNORECASE)\n    # Otherwise, treat the same as straight double quote\n    s = re.sub(r\"(\\w[^\\s']*)'\", r'\\1\u2019', s)\n    s = re.sub(r\"'(\\w)\", r'\u2018\\1', s)\n    if \"'\" in s: logging.warning(f\"couldn't convert straight single quote in [{s}]\")\n\n    return s\n\ndef clean_unicode(s):\n    s = s.replace('\\u00ad', '') # soft hyphen\n    s = s.replace('\\u2010', '-') # hyphen\n\n    # Some sources encode an i with an accent above using dotless i,\n    # which must be converted to normal i\n    s = list(s)\n    for i in range(len(s)-1):\n        # bug: we should only be looking for accents above, not\n        # below\n        if s[i] == '\u0131' and unicodedata.category(s[i+1]) == 'Mn':\n            s[i] = 'i'\n    s = ''.join(s)\n\n    # Selectively apply compatibility decomposition.\n    # This converts, e.g., \ufb01 to fi and \uff1a to :, but not \u00b2 to 2.\n    # Unsure: \u2026 to ...\n    # More classes could be added here.\n    def decompose(c):\n        d = unicodedata.decomposition(c)\n        if d and d.split(None, 1)[0] in ['<compat>', '<wide>', '<narrow>', '<noBreak>']:\n            return unicodedata.normalize('NFKD', c)\n        else:\n            return c\n    s = ''.join(map(decompose, s))\n\n    # Convert combining characters when possible\n    s = unicodedata.normalize('NFC', s)\n\n    return s\n\n\ndef normalize(oldnode, informat):\n    \"\"\"\n    Receives an XML 'paper' node and normalizes many of its fields, including:\n    - Unescaping HTML\n    - Normalizing quotes and other punctuation\n    - Mapping many characters to unicode\n    In addition, if the 'informat' is \"latex\", it will convert many LaTeX characters\n    to unicode equivalents. Note that these latter LaTeX operations are not idempotent.\n    \"\"\"\n\n    if oldnode.tag in ['url', 'href', 'mrf', 'doi', 'bibtype', 'bibkey',\n                       'revision', 'erratum', 'attachment', 'paper',\n                       'presentation', 'dataset', 'software', 'video']:\n        return\n    elif oldnode.tag in ['author', 'editor']:\n        for oldchild in oldnode:\n            normalize(oldchild, informat=informat)\n    else:\n        if informat == \"latex\":\n            if len(oldnode) > 0:\n                logging.error(\"field has child elements {}\".format(', '.join(child.tag for child in oldnode)))\n            oldtext = ''.join(oldnode.itertext())\n            newnode = latex_to_xml(oldtext, trivial_math=True, fixed_case=True)\n            newnode.tag = oldnode.tag\n            newnode.attrib.update(oldnode.attrib)\n            replace_node(oldnode, newnode)\n\n        maptext(oldnode, html.unescape)\n        maptext(oldnode, curly_quotes)\n        maptext(oldnode, clean_unicode)\n        if oldnode.tag in ['title', 'booktitle']:\n            protect(oldnode)\n\nif __name__ == \"__main__\":\n    import sys\n    import argparse\n    ap = argparse.ArgumentParser(description='Convert Anthology XML to standard format.')\n    ap.add_argument('infile', help=\"XML file to read\")\n    ap.add_argument('outfile', help=\"XML file to write\")\n    ap.add_argument('-t', '--latex', action=\"store_true\", help=\"Assume input fields are in LaTeX (not idempotent\")\n    args = ap.parse_args()\n\n    if args.latex:\n        informat = \"latex\"\n    else:\n        informat = \"xml\"\n\n    tree = etree.parse(args.infile)\n    root = tree.getroot()\n    if not root.tail:\n        # lxml drops trailing newline\n        root.tail = \"\\n\"\n    for paper in root.findall('.//paper'):\n        papernum = \"{} vol {} paper {}\".format(root.attrib['id'],\n                                               paper.getparent().attrib['id'],\n                                               paper.attrib['id'])\n        for oldnode in paper:\n            location = \"{}:{}\".format(papernum, oldnode.tag)\n            normalize(oldnode, informat=informat)\n\n    tree.write(args.outfile, encoding=\"UTF-8\", xml_declaration=True)\n", "path": "bin/normalize_anth.py"}, {"content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright 2019 Matt Post <post@cs.jhu.edu>\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nIngests data into the Anthology. It takes an XML file and:\n\n- converts to nested format if not already done\n- applies normalization (fixed-case protection)\n- runs sanity checks (e.g., % of authors in input data that are new)\n\"\"\"\n\nimport argparse\nimport os\nimport re\nimport sys\n\nimport lxml.etree as etree\n\nfrom datetime import datetime\n\nfrom normalize_anth import normalize\nfrom anthology.utils import make_nested, make_simple_element, build_anthology_id, indent\nfrom anthology.index import AnthologyIndex\nfrom anthology.people import PersonName\n\nfrom itertools import chain\n\n\nif __name__ == '__main__':\n    now = datetime.now()\n    today = f'{now.year}-{now.month:02d}-{now.day:02d}'\n\n    parser = argparse.ArgumentParser()\n    parser.add_argument('infile')\n    parser.add_argument('--ingest-date', '-d', type=str, default=today, help='Ingestion date as YYYY-MM-DD. Default: %(default)s.')\n    parser.add_argument('--append', '-a', action='store_true', help='Append to existing volume instead of quitting.')\n    args = parser.parse_args()\n\n    people = AnthologyIndex(None, srcdir=os.path.join(os.path.dirname(sys.argv[0]), '..', 'data'))\n\n    pdf_directory = os.path.dirname(args.infile)\n    tree_being_added = etree.parse(args.infile)\n\n    # Ensure nested format\n    root_being_added = make_nested(tree_being_added.getroot(), pdf_path=os.path.dirname(args.infile))\n    collection_id = root_being_added.attrib['id']\n\n    # Ensure names are properly identified\n    ambiguous = {}\n    for paper in root_being_added.findall('.//paper'):\n        anth_id = build_anthology_id(collection_id,\n                                     paper.getparent().attrib['id'],\n                                     paper.attrib['id'])\n\n        for node in chain(paper.findall('author'), paper.findall('editor')):\n            name = PersonName.from_element(node)\n            ids = people.get_ids(name)\n            if len(ids) > 1:\n                print(f'WARNING ({anth_id}): ambiguous author {name}, defaulting to first of {ids}')\n                ambiguous[anth_id] = (name, ids)\n\n                node.attrib['id'] = ids[0]\n\n        # Ensure PDF exists. PDF should be in the same directory as the XML file being ingested.\n        pdf_path = os.path.join(pdf_directory, f'{anth_id}.pdf')\n        if not os.path.exists(pdf_path):\n            print(f'FATAL: cannot file PDF {pdf_path}!')\n            sys.exit(1)\n\n    # Normalize\n    for paper in root_being_added.findall('.//paper'):\n        for oldnode in paper:\n            normalize(oldnode, informat='latex')\n\n    # Ingest each volume.\n    # First, find the XML file.\n    collection_file = os.path.join(os.path.dirname(sys.argv[0]), '..', 'data', 'xml', f'{collection_id}.xml')\n\n    if os.path.exists(collection_file):\n        existing_tree = etree.parse(collection_file)\n    else:\n        existing_tree = etree.ElementTree(make_simple_element('collection', attrib={'id': collection_id}))\n\n    # Insert each volume\n    for i, new_volume in enumerate(root_being_added.findall('volume')):\n        new_volume_id = int(new_volume.attrib['id'])\n        existing_volume = existing_tree.getroot().find(f\"./volume[@id='{new_volume_id}']\")\n        if existing_volume is None:\n            new_volume.attrib['ingest-date'] = args.ingest_date\n\n            # Find the insertion point among the other volumes\n            insertion_point = 0\n            for i, volume in enumerate(existing_tree.getroot()):\n                if new_volume_id < int(volume.attrib['id']):\n                    break\n                insertion_point = i + 1\n            print(f\"Inserting volume {new_volume_id} at collection position {insertion_point}\")\n            existing_tree.getroot().insert(insertion_point, new_volume)\n        else:\n            # Append to existing volume (useful for TACL, which has a single volume each year) if requested\n            if args.append:\n                for paper in new_volume.findall('./paper'):\n                    print(f'Appending {paper.attrib[\"id\"]}')\n                    paper.attrib['ingest-date'] = args.ingest_date\n                    existing_volume.append(paper)\n            else:\n                print(f'Volume {new_volume_id} has already been inserted into {collection_file}.')\n                print(f'You can append to this volume by passing `--append` (or `-a`) to this script.')\n                print(f'Quitting, since you didn\\'t.')\n                sys.exit(1)\n\n    indent(existing_tree.getroot())\n    existing_tree.write(collection_file, encoding='UTF-8', xml_declaration=True, with_tail=True)\n", "path": "bin/ingest.py"}]}