We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Fix BWB Importbot Low Quality records
<!-- What problem are we solving? What does the experience look like today? What are the symptoms? -->
Importbot is importing low-quality records from unspecified source. Titles are public domain reprints (possibly print on demand) with keyword-stuffed titles. 

### Evidence / Screenshot (if possible)
<img width="858" alt="Screenshot 2020-11-24 at 21 31 56" src="https://user-images.githubusercontent.com/17739465/100149057-e87a3b80-2e9d-11eb-9291-b49854aa65b7.png">

Example: https://openlibrary.org/search?q=adventures+of+sherlock+holmes+conan+doyle&mode=everything&page=3

### Proposal & Constraints
<!-- What is the proposed solution / implementation? Is there a precedent of this approach succeeding elsewhere? -->
Books have valid ISBNs and human editors should be allowed to add manually but these should be blocked from import by importbot.

### Related files
<!-- Files related to this issue; this is super useful for new contributors who might want to help! If you're not sure, leave this blank; a maintainer will add them. -->

### Stakeholders
<!-- @ tag stakeholders of this bug -->
@mekarpeles 

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scripts/partner_batch_imports.py`
Content:
```
1 """
2 Process partner bibliographic csv data into importable json book
3 records and then batch submit into the ImportBot
4 `import_item` table (http://openlibrary.org/admin/imports)
5 which queues items to be imported via the
6 Open Library JSON import API: https://openlibrary.org/api/import
7 
8 To Run:
9 
10 PYTHONPATH=. python ./scripts/partner_batch_imports.py /olsystem/etc/openlibrary.yml
11 """
12 
13 import os
14 import re
15 import sys
16 import web
17 import datetime
18 from datetime import timedelta
19 import logging
20 import requests
21 
22 from infogami import config  # noqa: F401
23 from openlibrary.config import load_config
24 from openlibrary.core.imports import Batch
25 from scripts.solr_builder.solr_builder.fn_to_cli import FnToCLI
26 
27 logger = logging.getLogger("openlibrary.importer.bwb")
28 
29 SCHEMA_URL = (
30     "https://raw.githubusercontent.com/internetarchive"
31     "/openlibrary-client/master/olclient/schemata/import.schema.json"
32 )
33 
34 
35 class Biblio:
36 
37     ACTIVE_FIELDS = [
38         'title',
39         'isbn_13',
40         'publish_date',
41         'publishers',
42         'weight',
43         'authors',
44         'lc_classifications',
45         'pagination',
46         'languages',
47         'subjects',
48         'source_records',
49     ]
50     INACTIVE_FIELDS = [
51         "copyright",
52         "issn",
53         "doi",
54         "lccn",
55         "dewey",
56         "length",
57         "width",
58         "height",
59     ]
60     REQUIRED_FIELDS = requests.get(SCHEMA_URL).json()['required']
61 
62     NONBOOK = """A2 AA AB AJ AVI AZ BK BM C3 CD CE CF CR CRM CRW CX D3 DA DD DF DI DL DO DR
63     DRM DRW DS DV EC FC FI FM FR FZ GB GC GM GR H3 H5 L3 L5 LP MAC MC MF MG MH ML MS MSX MZ
64     N64 NGA NGB NGC NGE NT OR OS PC PP PRP PS PSC PY QU RE RV SA SD SG SH SK SL SMD SN SO SO1
65     SO2 SR SU TA TB TR TS TY UX V35 V8 VC VD VE VF VK VM VN VO VP VS VU VY VZ WA WC WI WL WM
66     WP WT WX XL XZ ZF ZZ""".split()
67 
68     def __init__(self, data):
69         self.isbn = data[124]
70         self.source_id = 'bwb:%s' % self.isbn
71         self.isbn_13 = [self.isbn]
72         self.title = data[10]
73         self.primary_format = data[6]
74         self.publish_date = data[20][:4]  # YYYY, YYYYMMDD
75         self.publishers = [data[135]]
76         self.weight = data[39]
77         self.authors = self.contributors(data)
78         self.lc_classifications = [data[147]] if data[147] else []
79         self.pagination = data[36]
80         self.languages = [data[37].lower()]
81         self.source_records = [self.source_id]
82         self.subjects = [
83             s.capitalize().replace('_', ', ')
84             for s in data[91:100]
85             # + data[101:120]
86             # + data[153:158]
87             if s
88         ]
89 
90         # Inactive fields
91         self.copyright = data[19]
92         self.issn = data[54]
93         self.doi = data[145]
94         self.lccn = data[146]
95         self.dewey = data[49]
96         # physical_dimensions
97         # e.g. "5.4 x 4.7 x 0.2 inches"
98         self.length, self.width, self.height = data[40:43]
99 
100         # Assert importable
101         for field in self.REQUIRED_FIELDS + ['isbn_13']:
102             assert getattr(self, field), field
103         # This seems to be eliminating books too aggressively
104         #assert self.primary_format not in self.NONBOOK, f"{self.primary_format} is NONBOOK"
105 
106     @staticmethod
107     def contributors(data):
108         def make_author(name, _, typ):
109             author = {'name': name}
110             if typ == 'X':
111                 # set corporate contributor
112                 author['entity_type'] = 'org'
113             # TODO: sort out contributor types
114             # AU = author
115             # ED = editor
116             return author
117 
118         contributors = (
119             (data[21 + i * 3], data[22 + i * 3], data[23 + i * 3]) for i in range(5)
120         )
121 
122         # form list of author dicts
123         authors = [make_author(*c) for c in contributors if c[0]]
124         return authors
125 
126     def json(self):
127         return {
128             field: getattr(self, field)
129             for field in self.ACTIVE_FIELDS
130             if getattr(self, field)
131         }
132 
133 
134 def load_state(path, logfile):
135     """Retrieves starting point from logfile, if log exists
136 
137     Takes as input a path which expands to an ordered candidate list
138     of bettworldbks* filenames to process, the location of the
139     logfile, and determines which of those files are remaining, as
140     well as what our offset is in that file.
141 
142     e.g. if we request path containing f1, f2, f3 and our log
143     says f2,100 then we start our processing at f2 at the 100th line.
144 
145     This assumes the script is being called w/ e.g.:
146     /1/var/tmp/imports/2021-08/Bibliographic/*/
147     """
148     filenames = sorted(
149         os.path.join(path, f) for f in os.listdir(path) if f.startswith("bettworldbks")
150     )
151     try:
152         with open(logfile) as fin:
153             active_fname, offset = next(fin).strip().split(',')
154             unfinished_filenames = filenames[filenames.index(active_fname) :]
155             return unfinished_filenames, int(offset)
156     except (ValueError, OSError):
157         return filenames, 0
158 
159 
160 def update_state(logfile, fname, line_num=0):
161     """Records the last file we began processing and the current line"""
162     with open(logfile, 'w') as fout:
163         fout.write(f'{fname},{line_num}\n')
164 
165 
166 def csv_to_ol_json_item(line):
167     """converts a line to a book item"""
168     try:
169         data = line.decode().strip().split('|')
170     except UnicodeDecodeError:
171         data = line.decode('ISO-8859-1').strip().split('|')
172 
173     b = Biblio(data)
174     return {'ia_id': b.source_id, 'data': b.json()}
175 
176 
177 def batch_import(path, batch, batch_size=5000):
178     logfile = os.path.join(path, 'import.log')
179     filenames, offset = load_state(path, logfile)
180 
181     for fname in filenames:
182         book_items = []
183         with open(fname, 'rb') as f:
184             logger.info(f"Processing: {fname} from line {offset}")
185             for line_num, line in enumerate(f):
186 
187                 # skip over already processed records
188                 if offset:
189                     if offset > line_num:
190                         continue
191                     offset = 0
192 
193                 try:
194                     book_items.append(csv_to_ol_json_item(line))
195                 except AssertionError as e:
196                     logger.info(f"Error: {e} from {line}")
197 
198                 # If we have enough items, submit a batch
199                 if not ((line_num + 1) % batch_size):
200                     batch.add_items(book_items)
201                     update_state(logfile, fname, line_num)
202                     book_items = []  # clear added items
203 
204             # Add any remaining book_items to batch
205             if book_items:
206                 batch.add_items(book_items)
207             update_state(logfile, fname, line_num)
208 
209 
210 def main(ol_config: str, batch_path: str):
211     load_config(ol_config)
212 
213     # Partner data is offset ~15 days from start of month
214     date = datetime.date.today() - timedelta(days=15)
215     batch_name = "%s-%04d%02d" % ('bwb', date.year, date.month)
216     batch = Batch.find(batch_name) or Batch.new(batch_name)
217     batch_import(batch_path, batch)
218 
219 
220 if __name__ == '__main__':
221     FnToCLI(main).run()
222
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scripts/partner_batch_imports.py b/scripts/partner_batch_imports.py
--- a/scripts/partner_batch_imports.py
+++ b/scripts/partner_batch_imports.py
@@ -173,6 +173,10 @@
     b = Biblio(data)
     return {'ia_id': b.source_id, 'data': b.json()}
 
+def is_low_quality_book(book_item):
+    """check if a book item is of low quality"""
+    return ("notebook" in book_item.title.casefold() and "independently published" in book_item.publisher.casefold())
+
 
 def batch_import(path, batch, batch_size=5000):
     logfile = os.path.join(path, 'import.log')
@@ -191,7 +195,9 @@
                     offset = 0
 
                 try:
-                    book_items.append(csv_to_ol_json_item(line))
+                    book_item = csv_to_ol_json_item(line)
+                    if not is_low_quality_book(book_item["data"]):
+                        book_items.append(book_item)
                 except AssertionError as e:
                     logger.info(f"Error: {e} from {line}")
 
@@ -206,7 +212,6 @@
                 batch.add_items(book_items)
             update_state(logfile, fname, line_num)
 
-
 def main(ol_config: str, batch_path: str):
     load_config(ol_config)

{"golden_diff": "diff --git a/scripts/partner_batch_imports.py b/scripts/partner_batch_imports.py\n--- a/scripts/partner_batch_imports.py\n+++ b/scripts/partner_batch_imports.py\n@@ -173,6 +173,10 @@\n     b = Biblio(data)\n     return {'ia_id': b.source_id, 'data': b.json()}\n \n+def is_low_quality_book(book_item):\n+    \"\"\"check if a book item is of low quality\"\"\"\n+    return (\"notebook\" in book_item.title.casefold() and \"independently published\" in book_item.publisher.casefold())\n+\n \n def batch_import(path, batch, batch_size=5000):\n     logfile = os.path.join(path, 'import.log')\n@@ -191,7 +195,9 @@\n                     offset = 0\n \n                 try:\n-                    book_items.append(csv_to_ol_json_item(line))\n+                    book_item = csv_to_ol_json_item(line)\n+                    if not is_low_quality_book(book_item[\"data\"]):\n+                        book_items.append(book_item)\n                 except AssertionError as e:\n                     logger.info(f\"Error: {e} from {line}\")\n \n@@ -206,7 +212,6 @@\n                 batch.add_items(book_items)\n             update_state(logfile, fname, line_num)\n \n-\n def main(ol_config: str, batch_path: str):\n     load_config(ol_config)\n", "issue": "Fix BWB Importbot Low Quality records\n<!-- What problem are we solving? What does the experience look like today? What are the symptoms? -->\r\nImportbot is importing low-quality records from unspecified source. Titles are public domain reprints (possibly print on demand) with keyword-stuffed titles. \r\n\r\n### Evidence / Screenshot (if possible)\r\n<img width=\"858\" alt=\"Screenshot 2020-11-24 at 21 31 56\" src=\"https://user-images.githubusercontent.com/17739465/100149057-e87a3b80-2e9d-11eb-9291-b49854aa65b7.png\">\r\n\r\nExample: https://openlibrary.org/search?q=adventures+of+sherlock+holmes+conan+doyle&mode=everything&page=3\r\n\r\n### Proposal & Constraints\r\n<!-- What is the proposed solution / implementation? Is there a precedent of this approach succeeding elsewhere? -->\r\nBooks have valid ISBNs and human editors should be allowed to add manually but these should be blocked from import by importbot.\r\n\r\n### Related files\r\n<!-- Files related to this issue; this is super useful for new contributors who might want to help! If you're not sure, leave this blank; a maintainer will add them. -->\r\n\r\n### Stakeholders\r\n<!-- @ tag stakeholders of this bug -->\r\n@mekarpeles \n", "before_files": [{"content": "\"\"\"\nProcess partner bibliographic csv data into importable json book\nrecords and then batch submit into the ImportBot\n`import_item` table (http://openlibrary.org/admin/imports)\nwhich queues items to be imported via the\nOpen Library JSON import API: https://openlibrary.org/api/import\n\nTo Run:\n\nPYTHONPATH=. python ./scripts/partner_batch_imports.py /olsystem/etc/openlibrary.yml\n\"\"\"\n\nimport os\nimport re\nimport sys\nimport web\nimport datetime\nfrom datetime import timedelta\nimport logging\nimport requests\n\nfrom infogami import config  # noqa: F401\nfrom openlibrary.config import load_config\nfrom openlibrary.core.imports import Batch\nfrom scripts.solr_builder.solr_builder.fn_to_cli import FnToCLI\n\nlogger = logging.getLogger(\"openlibrary.importer.bwb\")\n\nSCHEMA_URL = (\n    \"https://raw.githubusercontent.com/internetarchive\"\n    \"/openlibrary-client/master/olclient/schemata/import.schema.json\"\n)\n\n\nclass Biblio:\n\n    ACTIVE_FIELDS = [\n        'title',\n        'isbn_13',\n        'publish_date',\n        'publishers',\n        'weight',\n        'authors',\n        'lc_classifications',\n        'pagination',\n        'languages',\n        'subjects',\n        'source_records',\n    ]\n    INACTIVE_FIELDS = [\n        \"copyright\",\n        \"issn\",\n        \"doi\",\n        \"lccn\",\n        \"dewey\",\n        \"length\",\n        \"width\",\n        \"height\",\n    ]\n    REQUIRED_FIELDS = requests.get(SCHEMA_URL).json()['required']\n\n    NONBOOK = \"\"\"A2 AA AB AJ AVI AZ BK BM C3 CD CE CF CR CRM CRW CX D3 DA DD DF DI DL DO DR\n    DRM DRW DS DV EC FC FI FM FR FZ GB GC GM GR H3 H5 L3 L5 LP MAC MC MF MG MH ML MS MSX MZ\n    N64 NGA NGB NGC NGE NT OR OS PC PP PRP PS PSC PY QU RE RV SA SD SG SH SK SL SMD SN SO SO1\n    SO2 SR SU TA TB TR TS TY UX V35 V8 VC VD VE VF VK VM VN VO VP VS VU VY VZ WA WC WI WL WM\n    WP WT WX XL XZ ZF ZZ\"\"\".split()\n\n    def __init__(self, data):\n        self.isbn = data[124]\n        self.source_id = 'bwb:%s' % self.isbn\n        self.isbn_13 = [self.isbn]\n        self.title = data[10]\n        self.primary_format = data[6]\n        self.publish_date = data[20][:4]  # YYYY, YYYYMMDD\n        self.publishers = [data[135]]\n        self.weight = data[39]\n        self.authors = self.contributors(data)\n        self.lc_classifications = [data[147]] if data[147] else []\n        self.pagination = data[36]\n        self.languages = [data[37].lower()]\n        self.source_records = [self.source_id]\n        self.subjects = [\n            s.capitalize().replace('_', ', ')\n            for s in data[91:100]\n            # + data[101:120]\n            # + data[153:158]\n            if s\n        ]\n\n        # Inactive fields\n        self.copyright = data[19]\n        self.issn = data[54]\n        self.doi = data[145]\n        self.lccn = data[146]\n        self.dewey = data[49]\n        # physical_dimensions\n        # e.g. \"5.4 x 4.7 x 0.2 inches\"\n        self.length, self.width, self.height = data[40:43]\n\n        # Assert importable\n        for field in self.REQUIRED_FIELDS + ['isbn_13']:\n            assert getattr(self, field), field\n        # This seems to be eliminating books too aggressively\n        #assert self.primary_format not in self.NONBOOK, f\"{self.primary_format} is NONBOOK\"\n\n    @staticmethod\n    def contributors(data):\n        def make_author(name, _, typ):\n            author = {'name': name}\n            if typ == 'X':\n                # set corporate contributor\n                author['entity_type'] = 'org'\n            # TODO: sort out contributor types\n            # AU = author\n            # ED = editor\n            return author\n\n        contributors = (\n            (data[21 + i * 3], data[22 + i * 3], data[23 + i * 3]) for i in range(5)\n        )\n\n        # form list of author dicts\n        authors = [make_author(*c) for c in contributors if c[0]]\n        return authors\n\n    def json(self):\n        return {\n            field: getattr(self, field)\n            for field in self.ACTIVE_FIELDS\n            if getattr(self, field)\n        }\n\n\ndef load_state(path, logfile):\n    \"\"\"Retrieves starting point from logfile, if log exists\n\n    Takes as input a path which expands to an ordered candidate list\n    of bettworldbks* filenames to process, the location of the\n    logfile, and determines which of those files are remaining, as\n    well as what our offset is in that file.\n\n    e.g. if we request path containing f1, f2, f3 and our log\n    says f2,100 then we start our processing at f2 at the 100th line.\n\n    This assumes the script is being called w/ e.g.:\n    /1/var/tmp/imports/2021-08/Bibliographic/*/\n    \"\"\"\n    filenames = sorted(\n        os.path.join(path, f) for f in os.listdir(path) if f.startswith(\"bettworldbks\")\n    )\n    try:\n        with open(logfile) as fin:\n            active_fname, offset = next(fin).strip().split(',')\n            unfinished_filenames = filenames[filenames.index(active_fname) :]\n            return unfinished_filenames, int(offset)\n    except (ValueError, OSError):\n        return filenames, 0\n\n\ndef update_state(logfile, fname, line_num=0):\n    \"\"\"Records the last file we began processing and the current line\"\"\"\n    with open(logfile, 'w') as fout:\n        fout.write(f'{fname},{line_num}\\n')\n\n\ndef csv_to_ol_json_item(line):\n    \"\"\"converts a line to a book item\"\"\"\n    try:\n        data = line.decode().strip().split('|')\n    except UnicodeDecodeError:\n        data = line.decode('ISO-8859-1').strip().split('|')\n\n    b = Biblio(data)\n    return {'ia_id': b.source_id, 'data': b.json()}\n\n\ndef batch_import(path, batch, batch_size=5000):\n    logfile = os.path.join(path, 'import.log')\n    filenames, offset = load_state(path, logfile)\n\n    for fname in filenames:\n        book_items = []\n        with open(fname, 'rb') as f:\n            logger.info(f\"Processing: {fname} from line {offset}\")\n            for line_num, line in enumerate(f):\n\n                # skip over already processed records\n                if offset:\n                    if offset > line_num:\n                        continue\n                    offset = 0\n\n                try:\n                    book_items.append(csv_to_ol_json_item(line))\n                except AssertionError as e:\n                    logger.info(f\"Error: {e} from {line}\")\n\n                # If we have enough items, submit a batch\n                if not ((line_num + 1) % batch_size):\n                    batch.add_items(book_items)\n                    update_state(logfile, fname, line_num)\n                    book_items = []  # clear added items\n\n            # Add any remaining book_items to batch\n            if book_items:\n                batch.add_items(book_items)\n            update_state(logfile, fname, line_num)\n\n\ndef main(ol_config: str, batch_path: str):\n    load_config(ol_config)\n\n    # Partner data is offset ~15 days from start of month\n    date = datetime.date.today() - timedelta(days=15)\n    batch_name = \"%s-%04d%02d\" % ('bwb', date.year, date.month)\n    batch = Batch.find(batch_name) or Batch.new(batch_name)\n    batch_import(batch_path, batch)\n\n\nif __name__ == '__main__':\n    FnToCLI(main).run()\n", "path": "scripts/partner_batch_imports.py"}], "after_files": [{"content": "\"\"\"\nProcess partner bibliographic csv data into importable json book\nrecords and then batch submit into the ImportBot\n`import_item` table (http://openlibrary.org/admin/imports)\nwhich queues items to be imported via the\nOpen Library JSON import API: https://openlibrary.org/api/import\n\nTo Run:\n\nPYTHONPATH=. python ./scripts/partner_batch_imports.py /olsystem/etc/openlibrary.yml\n\"\"\"\n\nimport os\nimport re\nimport sys\nimport web\nimport datetime\nfrom datetime import timedelta\nimport logging\nimport requests\n\nfrom infogami import config  # noqa: F401\nfrom openlibrary.config import load_config\nfrom openlibrary.core.imports import Batch\nfrom scripts.solr_builder.solr_builder.fn_to_cli import FnToCLI\n\nlogger = logging.getLogger(\"openlibrary.importer.bwb\")\n\nSCHEMA_URL = (\n    \"https://raw.githubusercontent.com/internetarchive\"\n    \"/openlibrary-client/master/olclient/schemata/import.schema.json\"\n)\n\n\nclass Biblio:\n\n    ACTIVE_FIELDS = [\n        'title',\n        'isbn_13',\n        'publish_date',\n        'publishers',\n        'weight',\n        'authors',\n        'lc_classifications',\n        'pagination',\n        'languages',\n        'subjects',\n        'source_records',\n    ]\n    INACTIVE_FIELDS = [\n        \"copyright\",\n        \"issn\",\n        \"doi\",\n        \"lccn\",\n        \"dewey\",\n        \"length\",\n        \"width\",\n        \"height\",\n    ]\n    REQUIRED_FIELDS = requests.get(SCHEMA_URL).json()['required']\n\n    NONBOOK = \"\"\"A2 AA AB AJ AVI AZ BK BM C3 CD CE CF CR CRM CRW CX D3 DA DD DF DI DL DO DR\n    DRM DRW DS DV EC FC FI FM FR FZ GB GC GM GR H3 H5 L3 L5 LP MAC MC MF MG MH ML MS MSX MZ\n    N64 NGA NGB NGC NGE NT OR OS PC PP PRP PS PSC PY QU RE RV SA SD SG SH SK SL SMD SN SO SO1\n    SO2 SR SU TA TB TR TS TY UX V35 V8 VC VD VE VF VK VM VN VO VP VS VU VY VZ WA WC WI WL WM\n    WP WT WX XL XZ ZF ZZ\"\"\".split()\n\n    def __init__(self, data):\n        self.isbn = data[124]\n        self.source_id = 'bwb:%s' % self.isbn\n        self.isbn_13 = [self.isbn]\n        self.title = data[10]\n        self.primary_format = data[6]\n        self.publish_date = data[20][:4]  # YYYY, YYYYMMDD\n        self.publishers = [data[135]]\n        self.weight = data[39]\n        self.authors = self.contributors(data)\n        self.lc_classifications = [data[147]] if data[147] else []\n        self.pagination = data[36]\n        self.languages = [data[37].lower()]\n        self.source_records = [self.source_id]\n        self.subjects = [\n            s.capitalize().replace('_', ', ')\n            for s in data[91:100]\n            # + data[101:120]\n            # + data[153:158]\n            if s\n        ]\n\n        # Inactive fields\n        self.copyright = data[19]\n        self.issn = data[54]\n        self.doi = data[145]\n        self.lccn = data[146]\n        self.dewey = data[49]\n        # physical_dimensions\n        # e.g. \"5.4 x 4.7 x 0.2 inches\"\n        self.length, self.width, self.height = data[40:43]\n\n        # Assert importable\n        for field in self.REQUIRED_FIELDS + ['isbn_13']:\n            assert getattr(self, field), field\n        # This seems to be eliminating books too aggressively\n        #assert self.primary_format not in self.NONBOOK, f\"{self.primary_format} is NONBOOK\"\n\n    @staticmethod\n    def contributors(data):\n        def make_author(name, _, typ):\n            author = {'name': name}\n            if typ == 'X':\n                # set corporate contributor\n                author['entity_type'] = 'org'\n            # TODO: sort out contributor types\n            # AU = author\n            # ED = editor\n            return author\n\n        contributors = (\n            (data[21 + i * 3], data[22 + i * 3], data[23 + i * 3]) for i in range(5)\n        )\n\n        # form list of author dicts\n        authors = [make_author(*c) for c in contributors if c[0]]\n        return authors\n\n    def json(self):\n        return {\n            field: getattr(self, field)\n            for field in self.ACTIVE_FIELDS\n            if getattr(self, field)\n        }\n\n\ndef load_state(path, logfile):\n    \"\"\"Retrieves starting point from logfile, if log exists\n\n    Takes as input a path which expands to an ordered candidate list\n    of bettworldbks* filenames to process, the location of the\n    logfile, and determines which of those files are remaining, as\n    well as what our offset is in that file.\n\n    e.g. if we request path containing f1, f2, f3 and our log\n    says f2,100 then we start our processing at f2 at the 100th line.\n\n    This assumes the script is being called w/ e.g.:\n    /1/var/tmp/imports/2021-08/Bibliographic/*/\n    \"\"\"\n    filenames = sorted(\n        os.path.join(path, f) for f in os.listdir(path) if f.startswith(\"bettworldbks\")\n    )\n    try:\n        with open(logfile) as fin:\n            active_fname, offset = next(fin).strip().split(',')\n            unfinished_filenames = filenames[filenames.index(active_fname) :]\n            return unfinished_filenames, int(offset)\n    except (ValueError, OSError):\n        return filenames, 0\n\n\ndef update_state(logfile, fname, line_num=0):\n    \"\"\"Records the last file we began processing and the current line\"\"\"\n    with open(logfile, 'w') as fout:\n        fout.write(f'{fname},{line_num}\\n')\n\n\ndef csv_to_ol_json_item(line):\n    \"\"\"converts a line to a book item\"\"\"\n    try:\n        data = line.decode().strip().split('|')\n    except UnicodeDecodeError:\n        data = line.decode('ISO-8859-1').strip().split('|')\n\n    b = Biblio(data)\n    return {'ia_id': b.source_id, 'data': b.json()}\n\ndef is_low_quality_book(book_item):\n    \"\"\"check if a book item is of low quality\"\"\"\n    return (\"notebook\" in book_item.title.casefold() and \"independently published\" in book_item.publisher.casefold())\n\n\ndef batch_import(path, batch, batch_size=5000):\n    logfile = os.path.join(path, 'import.log')\n    filenames, offset = load_state(path, logfile)\n\n    for fname in filenames:\n        book_items = []\n        with open(fname, 'rb') as f:\n            logger.info(f\"Processing: {fname} from line {offset}\")\n            for line_num, line in enumerate(f):\n\n                # skip over already processed records\n                if offset:\n                    if offset > line_num:\n                        continue\n                    offset = 0\n\n                try:\n                    book_item = csv_to_ol_json_item(line)\n                    if not is_low_quality_book(book_item[\"data\"]):\n                        book_items.append(book_item)\n                except AssertionError as e:\n                    logger.info(f\"Error: {e} from {line}\")\n\n                # If we have enough items, submit a batch\n                if not ((line_num + 1) % batch_size):\n                    batch.add_items(book_items)\n                    update_state(logfile, fname, line_num)\n                    book_items = []  # clear added items\n\n            # Add any remaining book_items to batch\n            if book_items:\n                batch.add_items(book_items)\n            update_state(logfile, fname, line_num)\n\ndef main(ol_config: str, batch_path: str):\n    load_config(ol_config)\n\n    # Partner data is offset ~15 days from start of month\n    date = datetime.date.today() - timedelta(days=15)\n    batch_name = \"%s-%04d%02d\" % ('bwb', date.year, date.month)\n    batch = Batch.find(batch_name) or Batch.new(batch_name)\n    batch_import(batch_path, batch)\n\n\nif __name__ == '__main__':\n    FnToCLI(main).run()\n", "path": "scripts/partner_batch_imports.py"}]}