We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
partner_batch_imports.py should not import books published in a future year
<!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
Per the [Open-Mic topics](https://docs.google.com/document/d/1LEbzsLZ1F9_YIQOoZzO7GoZnG1z-rudhZ9HNtsameTc/edit#heading=h.swvutwwydubf) for the Open Library Community call on 2022-11-29, we should not import partner data for books purporting to be published in a future year, as this is resulting in bad records of books that may never exist.

### Describe the problem that you'd like solved
<!-- A clear and concise description of what you want to happen. -->
When importing books, `partner_batch_imports.py` does not currently check if the `publish_date` is in a future year when importing. It should. E.g. if an import is attempted in the year 2022, it should not import a book purported to be published in 2023.

### Proposal & Constraints
<!-- What is the proposed solution / implementation? Is there a precedent of this approach succeeding elsewhere? -->
The proposed solution is to add a check to `batch_import()` in `partner_batch_imports.py` to ensure a book isn't purported to be published in a future year.

<!-- Which suggestions or requirements should be considered for how feature needs to appear or be implemented? -->

### Additional context
<!-- Add any other context or screenshots about the feature request here. -->
I will submit a PR to address this.

### Stakeholders
<!-- @ tag stakeholders of this bug -->
@mekarpeles, @cdrini 




--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scripts/partner_batch_imports.py`
Content:
```
1 """
2 Process partner bibliographic csv data into importable json book
3 records and then batch submit into the ImportBot
4 `import_item` table (http://openlibrary.org/admin/imports)
5 which queues items to be imported via the
6 Open Library JSON import API: https://openlibrary.org/api/import
7 
8 To Run:
9 
10 PYTHONPATH=. python ./scripts/partner_batch_imports.py /olsystem/etc/openlibrary.yml
11 """
12 
13 import datetime
14 import logging
15 import os
16 import re
17 
18 import requests
19 
20 from infogami import config  # noqa: F401
21 from openlibrary.config import load_config
22 from openlibrary.core.imports import Batch
23 from scripts.solr_builder.solr_builder.fn_to_cli import FnToCLI
24 
25 logger = logging.getLogger("openlibrary.importer.bwb")
26 
27 EXCLUDED_AUTHORS = {
28     x.casefold()
29     for x in (
30         "1570 publishing",
31         "bahija",
32         "bruna murino",
33         "creative elegant edition",
34         "delsee notebooks",
35         "grace garcia",
36         "holo",
37         "jeryx publishing",
38         "mado",
39         "mazzo",
40         "mikemix",
41         "mitch allison",
42         "pickleball publishing",
43         "pizzelle passion",
44         "punny cuaderno",
45         "razal koraya",
46         "t. d. publishing",
47         "tobias publishing",
48     )
49 }
50 
51 EXCLUDED_INDEPENDENTLY_PUBLISHED_TITLES = {
52     x.casefold()
53     for x in (
54         # Noisy classic re-prints
55         'annotated',
56         'annoté',
57         'classic',
58         'classics',
59         'illustarted',  # Some books have typos in their titles!
60         'illustrated',
61         'Illustrée',
62         'original',
63         'summary',
64         'version',
65         # Not a book
66         'calendar',
67         'diary',
68         'journal',
69         'logbook',
70         'notebook',
71         'notizbuch',
72         'planner',
73         'sketchbook',
74     )
75 }
76 
77 SCHEMA_URL = (
78     "https://raw.githubusercontent.com/internetarchive"
79     "/openlibrary-client/master/olclient/schemata/import.schema.json"
80 )
81 
82 
83 class Biblio:
84 
85     ACTIVE_FIELDS = [
86         'title',
87         'isbn_13',
88         'publish_date',
89         'publishers',
90         'weight',
91         'authors',
92         'lc_classifications',
93         'pagination',
94         'languages',
95         'subjects',
96         'source_records',
97     ]
98     INACTIVE_FIELDS = [
99         "copyright",
100         "issn",
101         "doi",
102         "lccn",
103         "dewey",
104         "length",
105         "width",
106         "height",
107     ]
108     REQUIRED_FIELDS = requests.get(SCHEMA_URL).json()['required']
109 
110     NONBOOK = """A2 AA AB AJ AVI AZ BK BM C3 CD CE CF CR CRM CRW CX D3 DA DD DF DI DL
111     DO DR DRM DRW DS DV EC FC FI FM FR FZ GB GC GM GR H3 H5 L3 L5 LP MAC MC MF MG MH ML
112     MS MSX MZ N64 NGA NGB NGC NGE NT OR OS PC PP PRP PS PSC PY QU RE RV SA SD SG SH SK
113     SL SMD SN SO SO1 SO2 SR SU TA TB TR TS TY UX V35 V8 VC VD VE VF VK VM VN VO VP VS
114     VU VY VZ WA WC WI WL WM WP WT WX XL XZ ZF ZZ""".split()
115 
116     def __init__(self, data):
117         self.isbn = data[124]
118         self.source_id = f'bwb:{self.isbn}'
119         self.isbn_13 = [self.isbn]
120         self.title = data[10]
121         self.primary_format = data[6]
122         self.publish_date = data[20][:4]  # YYYY, YYYYMMDD
123         self.publishers = [data[135]]
124         self.weight = data[39]
125         self.authors = self.contributors(data)
126         self.lc_classifications = [data[147]] if data[147] else []
127         self.pagination = data[36]
128         self.languages = [data[37].lower()]
129         self.source_records = [self.source_id]
130         self.subjects = [
131             s.capitalize().replace('_', ', ')
132             for s in data[91:100]
133             # + data[101:120]
134             # + data[153:158]
135             if s
136         ]
137 
138         # Inactive fields
139         self.copyright = data[19]
140         self.issn = data[54]
141         self.doi = data[145]
142         self.lccn = data[146]
143         self.dewey = data[49]
144         # physical_dimensions
145         # e.g. "5.4 x 4.7 x 0.2 inches"
146         self.length, self.width, self.height = data[40:43]
147 
148         # Assert importable
149         for field in self.REQUIRED_FIELDS + ['isbn_13']:
150             assert getattr(self, field), field
151         assert (
152             self.primary_format not in self.NONBOOK
153         ), f"{self.primary_format} is NONBOOK"
154 
155     @staticmethod
156     def contributors(data):
157         def make_author(name, _, typ):
158             author = {'name': name}
159             if typ == 'X':
160                 # set corporate contributor
161                 author['entity_type'] = 'org'
162             # TODO: sort out contributor types
163             # AU = author
164             # ED = editor
165             return author
166 
167         contributors = (
168             (data[21 + i * 3], data[22 + i * 3], data[23 + i * 3]) for i in range(5)
169         )
170 
171         # form list of author dicts
172         authors = [make_author(*c) for c in contributors if c[0]]
173         return authors
174 
175     def json(self):
176         return {
177             field: getattr(self, field)
178             for field in self.ACTIVE_FIELDS
179             if getattr(self, field)
180         }
181 
182 
183 def load_state(path, logfile):
184     """Retrieves starting point from logfile, if log exists
185 
186     Takes as input a path which expands to an ordered candidate list
187     of bettworldbks* filenames to process, the location of the
188     logfile, and determines which of those files are remaining, as
189     well as what our offset is in that file.
190 
191     e.g. if we request path containing f1, f2, f3 and our log
192     says f2,100 then we start our processing at f2 at the 100th line.
193 
194     This assumes the script is being called w/ e.g.:
195     /1/var/tmp/imports/2021-08/Bibliographic/*/
196     """
197     filenames = sorted(
198         os.path.join(path, f) for f in os.listdir(path) if f.startswith("bettworldbks")
199     )
200     try:
201         with open(logfile) as fin:
202             active_fname, offset = next(fin).strip().split(',')
203             unfinished_filenames = filenames[filenames.index(active_fname) :]
204             return unfinished_filenames, int(offset)
205     except (ValueError, OSError):
206         return filenames, 0
207 
208 
209 def update_state(logfile, fname, line_num=0):
210     """Records the last file we began processing and the current line"""
211     with open(logfile, 'w') as fout:
212         fout.write(f'{fname},{line_num}\n')
213 
214 
215 def csv_to_ol_json_item(line):
216     """converts a line to a book item"""
217     try:
218         data = line.decode().strip().split('|')
219     except UnicodeDecodeError:
220         data = line.decode('ISO-8859-1').strip().split('|')
221 
222     b = Biblio(data)
223     return {'ia_id': b.source_id, 'data': b.json()}
224 
225 
226 def is_low_quality_book(book_item) -> bool:
227     """
228     Check if a book item is of low quality which means that 1) one of its authors
229     (regardless of case) is in the set of excluded authors.
230     """
231     authors = {a['name'].casefold() for a in book_item.get('authors') or []}
232     if authors & EXCLUDED_AUTHORS:  # Leverage Python set intersection for speed.
233         return True
234 
235     # A recent independently published book with excluded key words in its title
236     # (regardless of case) is also considered a low quality book.
237     title_words = set(re.split(r'\W+', book_item["title"].casefold()))
238     publishers = {p.casefold() for p in book_item.get('publishers') or []}
239     publish_year = int(book_item.get("publish_date", "0")[:4])  # YYYY
240     return bool(
241         "independently published" in publishers
242         and publish_year >= 2018
243         and title_words & EXCLUDED_INDEPENDENTLY_PUBLISHED_TITLES
244     )
245 
246 
247 def batch_import(path, batch, batch_size=5000):
248     logfile = os.path.join(path, 'import.log')
249     filenames, offset = load_state(path, logfile)
250 
251     for fname in filenames:
252         book_items = []
253         with open(fname, 'rb') as f:
254             logger.info(f"Processing: {fname} from line {offset}")
255             for line_num, line in enumerate(f):
256 
257                 # skip over already processed records
258                 if offset:
259                     if offset > line_num:
260                         continue
261                     offset = 0
262 
263                 try:
264                     book_item = csv_to_ol_json_item(line)
265                     if not is_low_quality_book(book_item["data"]):
266                         book_items.append(book_item)
267                 except (AssertionError, IndexError) as e:
268                     logger.info(f"Error: {e} from {line}")
269 
270                 # If we have enough items, submit a batch
271                 if not ((line_num + 1) % batch_size):
272                     batch.add_items(book_items)
273                     update_state(logfile, fname, line_num)
274                     book_items = []  # clear added items
275 
276             # Add any remaining book_items to batch
277             if book_items:
278                 batch.add_items(book_items)
279             update_state(logfile, fname, line_num)
280 
281 
282 def main(ol_config: str, batch_path: str):
283     load_config(ol_config)
284 
285     # Partner data is offset ~15 days from start of month
286     date = datetime.date.today() - datetime.timedelta(days=15)
287     batch_name = "%s-%04d%02d" % ('bwb', date.year, date.month)
288     batch = Batch.find(batch_name) or Batch.new(batch_name)
289     batch_import(batch_path, batch)
290 
291 
292 if __name__ == '__main__':
293     FnToCLI(main).run()
294
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scripts/partner_batch_imports.py b/scripts/partner_batch_imports.py
--- a/scripts/partner_batch_imports.py
+++ b/scripts/partner_batch_imports.py
@@ -10,10 +10,12 @@
 PYTHONPATH=. python ./scripts/partner_batch_imports.py /olsystem/etc/openlibrary.yml
 """
 
+from collections.abc import Mapping
 import datetime
 import logging
 import os
 import re
+from typing import TypedDict, cast
 
 import requests
 
@@ -244,6 +246,18 @@
     )
 
 
+def is_published_in_future_year(book_item: Mapping[str, str | list]) -> bool:
+    """
+    Prevent import of books with a publication after the current year.
+
+    Some import sources have publication dates in a future year, and the likelihood
+    is high that this is bad data. So we don't want to import these.
+    """
+    publish_year = int(cast(str, book_item.get("publish_date", "0")[:4]))  # YYYY
+    this_year = datetime.datetime.now().year
+    return publish_year > this_year
+
+
 def batch_import(path, batch, batch_size=5000):
     logfile = os.path.join(path, 'import.log')
     filenames, offset = load_state(path, logfile)
@@ -262,7 +276,12 @@
 
                 try:
                     book_item = csv_to_ol_json_item(line)
-                    if not is_low_quality_book(book_item["data"]):
+                    if not any(
+                        [
+                            is_low_quality_book(book_item["data"]),
+                            is_published_in_future_year(book_item["data"]),
+                        ]
+                    ):
                         book_items.append(book_item)
                 except (AssertionError, IndexError) as e:
                     logger.info(f"Error: {e} from {line}")

{"golden_diff": "diff --git a/scripts/partner_batch_imports.py b/scripts/partner_batch_imports.py\n--- a/scripts/partner_batch_imports.py\n+++ b/scripts/partner_batch_imports.py\n@@ -10,10 +10,12 @@\n PYTHONPATH=. python ./scripts/partner_batch_imports.py /olsystem/etc/openlibrary.yml\n \"\"\"\n \n+from collections.abc import Mapping\n import datetime\n import logging\n import os\n import re\n+from typing import TypedDict, cast\n \n import requests\n \n@@ -244,6 +246,18 @@\n     )\n \n \n+def is_published_in_future_year(book_item: Mapping[str, str | list]) -> bool:\n+    \"\"\"\n+    Prevent import of books with a publication after the current year.\n+\n+    Some import sources have publication dates in a future year, and the likelihood\n+    is high that this is bad data. So we don't want to import these.\n+    \"\"\"\n+    publish_year = int(cast(str, book_item.get(\"publish_date\", \"0\")[:4]))  # YYYY\n+    this_year = datetime.datetime.now().year\n+    return publish_year > this_year\n+\n+\n def batch_import(path, batch, batch_size=5000):\n     logfile = os.path.join(path, 'import.log')\n     filenames, offset = load_state(path, logfile)\n@@ -262,7 +276,12 @@\n \n                 try:\n                     book_item = csv_to_ol_json_item(line)\n-                    if not is_low_quality_book(book_item[\"data\"]):\n+                    if not any(\n+                        [\n+                            is_low_quality_book(book_item[\"data\"]),\n+                            is_published_in_future_year(book_item[\"data\"]),\n+                        ]\n+                    ):\n                         book_items.append(book_item)\n                 except (AssertionError, IndexError) as e:\n                     logger.info(f\"Error: {e} from {line}\")\n", "issue": "partner_batch_imports.py should not import books published in a future year\n<!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->\r\nPer the [Open-Mic topics](https://docs.google.com/document/d/1LEbzsLZ1F9_YIQOoZzO7GoZnG1z-rudhZ9HNtsameTc/edit#heading=h.swvutwwydubf) for the Open Library Community call on 2022-11-29, we should not import partner data for books purporting to be published in a future year, as this is resulting in bad records of books that may never exist.\r\n\r\n### Describe the problem that you'd like solved\r\n<!-- A clear and concise description of what you want to happen. -->\r\nWhen importing books, `partner_batch_imports.py` does not currently check if the `publish_date` is in a future year when importing. It should. E.g. if an import is attempted in the year 2022, it should not import a book purported to be published in 2023.\r\n\r\n### Proposal & Constraints\r\n<!-- What is the proposed solution / implementation? Is there a precedent of this approach succeeding elsewhere? -->\r\nThe proposed solution is to add a check to `batch_import()` in `partner_batch_imports.py` to ensure a book isn't purported to be published in a future year.\r\n\r\n<!-- Which suggestions or requirements should be considered for how feature needs to appear or be implemented? -->\r\n\r\n### Additional context\r\n<!-- Add any other context or screenshots about the feature request here. -->\r\nI will submit a PR to address this.\r\n\r\n### Stakeholders\r\n<!-- @ tag stakeholders of this bug -->\r\n@mekarpeles, @cdrini \r\n\r\n\r\n\n", "before_files": [{"content": "\"\"\"\nProcess partner bibliographic csv data into importable json book\nrecords and then batch submit into the ImportBot\n`import_item` table (http://openlibrary.org/admin/imports)\nwhich queues items to be imported via the\nOpen Library JSON import API: https://openlibrary.org/api/import\n\nTo Run:\n\nPYTHONPATH=. python ./scripts/partner_batch_imports.py /olsystem/etc/openlibrary.yml\n\"\"\"\n\nimport datetime\nimport logging\nimport os\nimport re\n\nimport requests\n\nfrom infogami import config  # noqa: F401\nfrom openlibrary.config import load_config\nfrom openlibrary.core.imports import Batch\nfrom scripts.solr_builder.solr_builder.fn_to_cli import FnToCLI\n\nlogger = logging.getLogger(\"openlibrary.importer.bwb\")\n\nEXCLUDED_AUTHORS = {\n    x.casefold()\n    for x in (\n        \"1570 publishing\",\n        \"bahija\",\n        \"bruna murino\",\n        \"creative elegant edition\",\n        \"delsee notebooks\",\n        \"grace garcia\",\n        \"holo\",\n        \"jeryx publishing\",\n        \"mado\",\n        \"mazzo\",\n        \"mikemix\",\n        \"mitch allison\",\n        \"pickleball publishing\",\n        \"pizzelle passion\",\n        \"punny cuaderno\",\n        \"razal koraya\",\n        \"t. d. publishing\",\n        \"tobias publishing\",\n    )\n}\n\nEXCLUDED_INDEPENDENTLY_PUBLISHED_TITLES = {\n    x.casefold()\n    for x in (\n        # Noisy classic re-prints\n        'annotated',\n        'annot\u00e9',\n        'classic',\n        'classics',\n        'illustarted',  # Some books have typos in their titles!\n        'illustrated',\n        'Illustr\u00e9e',\n        'original',\n        'summary',\n        'version',\n        # Not a book\n        'calendar',\n        'diary',\n        'journal',\n        'logbook',\n        'notebook',\n        'notizbuch',\n        'planner',\n        'sketchbook',\n    )\n}\n\nSCHEMA_URL = (\n    \"https://raw.githubusercontent.com/internetarchive\"\n    \"/openlibrary-client/master/olclient/schemata/import.schema.json\"\n)\n\n\nclass Biblio:\n\n    ACTIVE_FIELDS = [\n        'title',\n        'isbn_13',\n        'publish_date',\n        'publishers',\n        'weight',\n        'authors',\n        'lc_classifications',\n        'pagination',\n        'languages',\n        'subjects',\n        'source_records',\n    ]\n    INACTIVE_FIELDS = [\n        \"copyright\",\n        \"issn\",\n        \"doi\",\n        \"lccn\",\n        \"dewey\",\n        \"length\",\n        \"width\",\n        \"height\",\n    ]\n    REQUIRED_FIELDS = requests.get(SCHEMA_URL).json()['required']\n\n    NONBOOK = \"\"\"A2 AA AB AJ AVI AZ BK BM C3 CD CE CF CR CRM CRW CX D3 DA DD DF DI DL\n    DO DR DRM DRW DS DV EC FC FI FM FR FZ GB GC GM GR H3 H5 L3 L5 LP MAC MC MF MG MH ML\n    MS MSX MZ N64 NGA NGB NGC NGE NT OR OS PC PP PRP PS PSC PY QU RE RV SA SD SG SH SK\n    SL SMD SN SO SO1 SO2 SR SU TA TB TR TS TY UX V35 V8 VC VD VE VF VK VM VN VO VP VS\n    VU VY VZ WA WC WI WL WM WP WT WX XL XZ ZF ZZ\"\"\".split()\n\n    def __init__(self, data):\n        self.isbn = data[124]\n        self.source_id = f'bwb:{self.isbn}'\n        self.isbn_13 = [self.isbn]\n        self.title = data[10]\n        self.primary_format = data[6]\n        self.publish_date = data[20][:4]  # YYYY, YYYYMMDD\n        self.publishers = [data[135]]\n        self.weight = data[39]\n        self.authors = self.contributors(data)\n        self.lc_classifications = [data[147]] if data[147] else []\n        self.pagination = data[36]\n        self.languages = [data[37].lower()]\n        self.source_records = [self.source_id]\n        self.subjects = [\n            s.capitalize().replace('_', ', ')\n            for s in data[91:100]\n            # + data[101:120]\n            # + data[153:158]\n            if s\n        ]\n\n        # Inactive fields\n        self.copyright = data[19]\n        self.issn = data[54]\n        self.doi = data[145]\n        self.lccn = data[146]\n        self.dewey = data[49]\n        # physical_dimensions\n        # e.g. \"5.4 x 4.7 x 0.2 inches\"\n        self.length, self.width, self.height = data[40:43]\n\n        # Assert importable\n        for field in self.REQUIRED_FIELDS + ['isbn_13']:\n            assert getattr(self, field), field\n        assert (\n            self.primary_format not in self.NONBOOK\n        ), f\"{self.primary_format} is NONBOOK\"\n\n    @staticmethod\n    def contributors(data):\n        def make_author(name, _, typ):\n            author = {'name': name}\n            if typ == 'X':\n                # set corporate contributor\n                author['entity_type'] = 'org'\n            # TODO: sort out contributor types\n            # AU = author\n            # ED = editor\n            return author\n\n        contributors = (\n            (data[21 + i * 3], data[22 + i * 3], data[23 + i * 3]) for i in range(5)\n        )\n\n        # form list of author dicts\n        authors = [make_author(*c) for c in contributors if c[0]]\n        return authors\n\n    def json(self):\n        return {\n            field: getattr(self, field)\n            for field in self.ACTIVE_FIELDS\n            if getattr(self, field)\n        }\n\n\ndef load_state(path, logfile):\n    \"\"\"Retrieves starting point from logfile, if log exists\n\n    Takes as input a path which expands to an ordered candidate list\n    of bettworldbks* filenames to process, the location of the\n    logfile, and determines which of those files are remaining, as\n    well as what our offset is in that file.\n\n    e.g. if we request path containing f1, f2, f3 and our log\n    says f2,100 then we start our processing at f2 at the 100th line.\n\n    This assumes the script is being called w/ e.g.:\n    /1/var/tmp/imports/2021-08/Bibliographic/*/\n    \"\"\"\n    filenames = sorted(\n        os.path.join(path, f) for f in os.listdir(path) if f.startswith(\"bettworldbks\")\n    )\n    try:\n        with open(logfile) as fin:\n            active_fname, offset = next(fin).strip().split(',')\n            unfinished_filenames = filenames[filenames.index(active_fname) :]\n            return unfinished_filenames, int(offset)\n    except (ValueError, OSError):\n        return filenames, 0\n\n\ndef update_state(logfile, fname, line_num=0):\n    \"\"\"Records the last file we began processing and the current line\"\"\"\n    with open(logfile, 'w') as fout:\n        fout.write(f'{fname},{line_num}\\n')\n\n\ndef csv_to_ol_json_item(line):\n    \"\"\"converts a line to a book item\"\"\"\n    try:\n        data = line.decode().strip().split('|')\n    except UnicodeDecodeError:\n        data = line.decode('ISO-8859-1').strip().split('|')\n\n    b = Biblio(data)\n    return {'ia_id': b.source_id, 'data': b.json()}\n\n\ndef is_low_quality_book(book_item) -> bool:\n    \"\"\"\n    Check if a book item is of low quality which means that 1) one of its authors\n    (regardless of case) is in the set of excluded authors.\n    \"\"\"\n    authors = {a['name'].casefold() for a in book_item.get('authors') or []}\n    if authors & EXCLUDED_AUTHORS:  # Leverage Python set intersection for speed.\n        return True\n\n    # A recent independently published book with excluded key words in its title\n    # (regardless of case) is also considered a low quality book.\n    title_words = set(re.split(r'\\W+', book_item[\"title\"].casefold()))\n    publishers = {p.casefold() for p in book_item.get('publishers') or []}\n    publish_year = int(book_item.get(\"publish_date\", \"0\")[:4])  # YYYY\n    return bool(\n        \"independently published\" in publishers\n        and publish_year >= 2018\n        and title_words & EXCLUDED_INDEPENDENTLY_PUBLISHED_TITLES\n    )\n\n\ndef batch_import(path, batch, batch_size=5000):\n    logfile = os.path.join(path, 'import.log')\n    filenames, offset = load_state(path, logfile)\n\n    for fname in filenames:\n        book_items = []\n        with open(fname, 'rb') as f:\n            logger.info(f\"Processing: {fname} from line {offset}\")\n            for line_num, line in enumerate(f):\n\n                # skip over already processed records\n                if offset:\n                    if offset > line_num:\n                        continue\n                    offset = 0\n\n                try:\n                    book_item = csv_to_ol_json_item(line)\n                    if not is_low_quality_book(book_item[\"data\"]):\n                        book_items.append(book_item)\n                except (AssertionError, IndexError) as e:\n                    logger.info(f\"Error: {e} from {line}\")\n\n                # If we have enough items, submit a batch\n                if not ((line_num + 1) % batch_size):\n                    batch.add_items(book_items)\n                    update_state(logfile, fname, line_num)\n                    book_items = []  # clear added items\n\n            # Add any remaining book_items to batch\n            if book_items:\n                batch.add_items(book_items)\n            update_state(logfile, fname, line_num)\n\n\ndef main(ol_config: str, batch_path: str):\n    load_config(ol_config)\n\n    # Partner data is offset ~15 days from start of month\n    date = datetime.date.today() - datetime.timedelta(days=15)\n    batch_name = \"%s-%04d%02d\" % ('bwb', date.year, date.month)\n    batch = Batch.find(batch_name) or Batch.new(batch_name)\n    batch_import(batch_path, batch)\n\n\nif __name__ == '__main__':\n    FnToCLI(main).run()\n", "path": "scripts/partner_batch_imports.py"}], "after_files": [{"content": "\"\"\"\nProcess partner bibliographic csv data into importable json book\nrecords and then batch submit into the ImportBot\n`import_item` table (http://openlibrary.org/admin/imports)\nwhich queues items to be imported via the\nOpen Library JSON import API: https://openlibrary.org/api/import\n\nTo Run:\n\nPYTHONPATH=. python ./scripts/partner_batch_imports.py /olsystem/etc/openlibrary.yml\n\"\"\"\n\nfrom collections.abc import Mapping\nimport datetime\nimport logging\nimport os\nimport re\nfrom typing import TypedDict, cast\n\nimport requests\n\nfrom infogami import config  # noqa: F401\nfrom openlibrary.config import load_config\nfrom openlibrary.core.imports import Batch\nfrom scripts.solr_builder.solr_builder.fn_to_cli import FnToCLI\n\nlogger = logging.getLogger(\"openlibrary.importer.bwb\")\n\nEXCLUDED_AUTHORS = {\n    x.casefold()\n    for x in (\n        \"1570 publishing\",\n        \"bahija\",\n        \"bruna murino\",\n        \"creative elegant edition\",\n        \"delsee notebooks\",\n        \"grace garcia\",\n        \"holo\",\n        \"jeryx publishing\",\n        \"mado\",\n        \"mazzo\",\n        \"mikemix\",\n        \"mitch allison\",\n        \"pickleball publishing\",\n        \"pizzelle passion\",\n        \"punny cuaderno\",\n        \"razal koraya\",\n        \"t. d. publishing\",\n        \"tobias publishing\",\n    )\n}\n\nEXCLUDED_INDEPENDENTLY_PUBLISHED_TITLES = {\n    x.casefold()\n    for x in (\n        # Noisy classic re-prints\n        'annotated',\n        'annot\u00e9',\n        'classic',\n        'classics',\n        'illustarted',  # Some books have typos in their titles!\n        'illustrated',\n        'Illustr\u00e9e',\n        'original',\n        'summary',\n        'version',\n        # Not a book\n        'calendar',\n        'diary',\n        'journal',\n        'logbook',\n        'notebook',\n        'notizbuch',\n        'planner',\n        'sketchbook',\n    )\n}\n\nSCHEMA_URL = (\n    \"https://raw.githubusercontent.com/internetarchive\"\n    \"/openlibrary-client/master/olclient/schemata/import.schema.json\"\n)\n\n\nclass Biblio:\n\n    ACTIVE_FIELDS = [\n        'title',\n        'isbn_13',\n        'publish_date',\n        'publishers',\n        'weight',\n        'authors',\n        'lc_classifications',\n        'pagination',\n        'languages',\n        'subjects',\n        'source_records',\n    ]\n    INACTIVE_FIELDS = [\n        \"copyright\",\n        \"issn\",\n        \"doi\",\n        \"lccn\",\n        \"dewey\",\n        \"length\",\n        \"width\",\n        \"height\",\n    ]\n    REQUIRED_FIELDS = requests.get(SCHEMA_URL).json()['required']\n\n    NONBOOK = \"\"\"A2 AA AB AJ AVI AZ BK BM C3 CD CE CF CR CRM CRW CX D3 DA DD DF DI DL\n    DO DR DRM DRW DS DV EC FC FI FM FR FZ GB GC GM GR H3 H5 L3 L5 LP MAC MC MF MG MH ML\n    MS MSX MZ N64 NGA NGB NGC NGE NT OR OS PC PP PRP PS PSC PY QU RE RV SA SD SG SH SK\n    SL SMD SN SO SO1 SO2 SR SU TA TB TR TS TY UX V35 V8 VC VD VE VF VK VM VN VO VP VS\n    VU VY VZ WA WC WI WL WM WP WT WX XL XZ ZF ZZ\"\"\".split()\n\n    def __init__(self, data):\n        self.isbn = data[124]\n        self.source_id = f'bwb:{self.isbn}'\n        self.isbn_13 = [self.isbn]\n        self.title = data[10]\n        self.primary_format = data[6]\n        self.publish_date = data[20][:4]  # YYYY, YYYYMMDD\n        self.publishers = [data[135]]\n        self.weight = data[39]\n        self.authors = self.contributors(data)\n        self.lc_classifications = [data[147]] if data[147] else []\n        self.pagination = data[36]\n        self.languages = [data[37].lower()]\n        self.source_records = [self.source_id]\n        self.subjects = [\n            s.capitalize().replace('_', ', ')\n            for s in data[91:100]\n            # + data[101:120]\n            # + data[153:158]\n            if s\n        ]\n\n        # Inactive fields\n        self.copyright = data[19]\n        self.issn = data[54]\n        self.doi = data[145]\n        self.lccn = data[146]\n        self.dewey = data[49]\n        # physical_dimensions\n        # e.g. \"5.4 x 4.7 x 0.2 inches\"\n        self.length, self.width, self.height = data[40:43]\n\n        # Assert importable\n        for field in self.REQUIRED_FIELDS + ['isbn_13']:\n            assert getattr(self, field), field\n        assert (\n            self.primary_format not in self.NONBOOK\n        ), f\"{self.primary_format} is NONBOOK\"\n\n    @staticmethod\n    def contributors(data):\n        def make_author(name, _, typ):\n            author = {'name': name}\n            if typ == 'X':\n                # set corporate contributor\n                author['entity_type'] = 'org'\n            # TODO: sort out contributor types\n            # AU = author\n            # ED = editor\n            return author\n\n        contributors = (\n            (data[21 + i * 3], data[22 + i * 3], data[23 + i * 3]) for i in range(5)\n        )\n\n        # form list of author dicts\n        authors = [make_author(*c) for c in contributors if c[0]]\n        return authors\n\n    def json(self):\n        return {\n            field: getattr(self, field)\n            for field in self.ACTIVE_FIELDS\n            if getattr(self, field)\n        }\n\n\ndef load_state(path, logfile):\n    \"\"\"Retrieves starting point from logfile, if log exists\n\n    Takes as input a path which expands to an ordered candidate list\n    of bettworldbks* filenames to process, the location of the\n    logfile, and determines which of those files are remaining, as\n    well as what our offset is in that file.\n\n    e.g. if we request path containing f1, f2, f3 and our log\n    says f2,100 then we start our processing at f2 at the 100th line.\n\n    This assumes the script is being called w/ e.g.:\n    /1/var/tmp/imports/2021-08/Bibliographic/*/\n    \"\"\"\n    filenames = sorted(\n        os.path.join(path, f) for f in os.listdir(path) if f.startswith(\"bettworldbks\")\n    )\n    try:\n        with open(logfile) as fin:\n            active_fname, offset = next(fin).strip().split(',')\n            unfinished_filenames = filenames[filenames.index(active_fname) :]\n            return unfinished_filenames, int(offset)\n    except (ValueError, OSError):\n        return filenames, 0\n\n\ndef update_state(logfile, fname, line_num=0):\n    \"\"\"Records the last file we began processing and the current line\"\"\"\n    with open(logfile, 'w') as fout:\n        fout.write(f'{fname},{line_num}\\n')\n\n\ndef csv_to_ol_json_item(line):\n    \"\"\"converts a line to a book item\"\"\"\n    try:\n        data = line.decode().strip().split('|')\n    except UnicodeDecodeError:\n        data = line.decode('ISO-8859-1').strip().split('|')\n\n    b = Biblio(data)\n    return {'ia_id': b.source_id, 'data': b.json()}\n\n\ndef is_low_quality_book(book_item) -> bool:\n    \"\"\"\n    Check if a book item is of low quality which means that 1) one of its authors\n    (regardless of case) is in the set of excluded authors.\n    \"\"\"\n    authors = {a['name'].casefold() for a in book_item.get('authors') or []}\n    if authors & EXCLUDED_AUTHORS:  # Leverage Python set intersection for speed.\n        return True\n\n    # A recent independently published book with excluded key words in its title\n    # (regardless of case) is also considered a low quality book.\n    title_words = set(re.split(r'\\W+', book_item[\"title\"].casefold()))\n    publishers = {p.casefold() for p in book_item.get('publishers') or []}\n    publish_year = int(book_item.get(\"publish_date\", \"0\")[:4])  # YYYY\n    return bool(\n        \"independently published\" in publishers\n        and publish_year >= 2018\n        and title_words & EXCLUDED_INDEPENDENTLY_PUBLISHED_TITLES\n    )\n\n\ndef is_published_in_future_year(book_item: Mapping[str, str | list]) -> bool:\n    \"\"\"\n    Prevent import of books with a publication after the current year.\n\n    Some import sources have publication dates in a future year, and the likelihood\n    is high that this is bad data. So we don't want to import these.\n    \"\"\"\n    publish_year = int(cast(str, book_item.get(\"publish_date\", \"0\")[:4]))  # YYYY\n    this_year = datetime.datetime.now().year\n    return publish_year > this_year\n\n\ndef batch_import(path, batch, batch_size=5000):\n    logfile = os.path.join(path, 'import.log')\n    filenames, offset = load_state(path, logfile)\n\n    for fname in filenames:\n        book_items = []\n        with open(fname, 'rb') as f:\n            logger.info(f\"Processing: {fname} from line {offset}\")\n            for line_num, line in enumerate(f):\n\n                # skip over already processed records\n                if offset:\n                    if offset > line_num:\n                        continue\n                    offset = 0\n\n                try:\n                    book_item = csv_to_ol_json_item(line)\n                    if not any(\n                        [\n                            is_low_quality_book(book_item[\"data\"]),\n                            is_published_in_future_year(book_item[\"data\"]),\n                        ]\n                    ):\n                        book_items.append(book_item)\n                except (AssertionError, IndexError) as e:\n                    logger.info(f\"Error: {e} from {line}\")\n\n                # If we have enough items, submit a batch\n                if not ((line_num + 1) % batch_size):\n                    batch.add_items(book_items)\n                    update_state(logfile, fname, line_num)\n                    book_items = []  # clear added items\n\n            # Add any remaining book_items to batch\n            if book_items:\n                batch.add_items(book_items)\n            update_state(logfile, fname, line_num)\n\n\ndef main(ol_config: str, batch_path: str):\n    load_config(ol_config)\n\n    # Partner data is offset ~15 days from start of month\n    date = datetime.date.today() - datetime.timedelta(days=15)\n    batch_name = \"%s-%04d%02d\" % ('bwb', date.year, date.month)\n    batch = Batch.find(batch_name) or Batch.new(batch_name)\n    batch_import(batch_path, batch)\n\n\nif __name__ == '__main__':\n    FnToCLI(main).run()\n", "path": "scripts/partner_batch_imports.py"}]}