We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
`providers/isbndb.py` crashes when the `'subjects'` value is a literal `None`
<!-- IMPORTANT: Before posting, be sure to redact or remove sensitive data, such as passwords, secret keys, session cookies, etc. -->
<!-- What problem are we solving? What does the experience look like today? What are the symptoms? -->

### Evidence / Screenshot (if possible)

    
Sometimes the `'subjects'` key in the ISBNdb dump have literal `None` values, which would cause an error even if doing something like `data.get('subjects', [])`, because it would return `None` instead of `[]`.
    
E.g., consider the following record:
```python
{
    'isbn': '0134878868',
    'msrp': '0.00',
    'title': 'Learn Adobe Premiere Pro Cc For Video Communication',
    'isbn13': '9780134878867',
    'authors': ['Dockery, Joe (author.)'],
    'binding': 'Electronic Resource',
    'language': 'en',
    'subjects': None,
}
```
This would crash with:
```python
        ^^^^^^^^^^^^^^^^^^^
  File "/openlibrary/scripts/providers/isbndb.py", line 73, in __init__
    self.subjects = [
                    ^
TypeError: 'NoneType' object is not iterable
```

### Stakeholders
<!-- @ tag stakeholders of this bug -->
@mekarpeles 

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scripts/providers/isbndb.py`
Content:
```
1 import re
2 import json
3 import logging
4 import os
5 from typing import Any, Final
6 import requests
7 
8 from json import JSONDecodeError
9 
10 from openlibrary.config import load_config
11 from openlibrary.core.imports import Batch
12 from openlibrary.plugins.upstream.utils import get_marc21_language
13 from scripts.partner_batch_imports import is_published_in_future_year
14 from scripts.solr_builder.solr_builder.fn_to_cli import FnToCLI
15 
16 logger = logging.getLogger("openlibrary.importer.isbndb")
17 
18 SCHEMA_URL = (
19     "https://raw.githubusercontent.com/internetarchive"
20     "/openlibrary-client/master/olclient/schemata/import.schema.json"
21 )
22 
23 NONBOOK: Final = ['dvd', 'dvd-rom', 'cd', 'cd-rom', 'cassette', 'sheet music', 'audio']
24 RE_YEAR = re.compile(r'(\d{4})')
25 
26 
27 def is_nonbook(binding: str, nonbooks: list[str]) -> bool:
28     """
29     Determine whether binding, or a substring of binding, split on " ", is
30     contained within nonbooks.
31     """
32     words = binding.split(" ")
33     return any(word.casefold() in nonbooks for word in words)
34 
35 
36 class ISBNdb:
37     ACTIVE_FIELDS = [
38         'authors',
39         'isbn_13',
40         'languages',
41         'number_of_pages',
42         'publish_date',
43         'publishers',
44         'source_records',
45         'subjects',
46         'title',
47     ]
48     INACTIVE_FIELDS = [
49         "copyright",
50         "dewey",
51         "doi",
52         "height",
53         "issn",
54         "lccn",
55         "length",
56         "width",
57         'lc_classifications',
58         'pagination',
59         'weight',
60     ]
61     REQUIRED_FIELDS = requests.get(SCHEMA_URL).json()['required']
62 
63     def __init__(self, data: dict[str, Any]):
64         self.isbn_13 = [data.get('isbn13')]
65         self.source_id = f'idb:{self.isbn_13[0]}'
66         self.title = data.get('title')
67         self.publish_date = self._get_year(data)  # 'YYYY'
68         self.publishers = self._get_list_if_present(data.get('publisher'))
69         self.authors = self.contributors(data)
70         self.number_of_pages = data.get('pages')
71         self.languages = self._get_languages(data)
72         self.source_records = [self.source_id]
73         self.subjects = [
74             subject.capitalize() for subject in data.get('subjects', '') if subject
75         ]
76         self.binding = data.get('binding', '')
77 
78         # Assert importable
79         for field in self.REQUIRED_FIELDS + ['isbn_13']:
80             assert getattr(self, field), field
81         assert is_nonbook(self.binding, NONBOOK) is False, "is_nonbook() returned True"
82         assert self.isbn_13 != [
83             "9780000000002"
84         ], f"known bad ISBN: {self.isbn_13}"  # TODO: this should do more than ignore one known-bad ISBN.
85 
86     def _get_languages(self, data: dict[str, Any]) -> list[str] | None:
87         """Extract a list of MARC 21 format languages from an ISBNDb JSONL line."""
88         language_line = data.get('language')
89         if not language_line:
90             return None
91 
92         possible_languages = re.split(',| |;', language_line)
93         unique_languages = []
94 
95         for language in possible_languages:
96             if (
97                 marc21_language := get_marc21_language(language)
98             ) and marc21_language not in unique_languages:
99                 unique_languages.append(marc21_language)
100 
101         return unique_languages or None
102 
103     def _get_list_if_present(self, item: str | None) -> list[str] | None:
104         """Return items as a list, or None."""
105         return [item] if item else None
106 
107     def _get_year(self, data: dict[str, Any]) -> str | None:
108         """Return a year str/int as a four digit string, or None."""
109         result = ""
110         if publish_date := data.get('date_published'):
111             if isinstance(publish_date, str):
112                 m = RE_YEAR.search(publish_date)
113                 result = m.group(1) if m else None  # type: ignore[assignment]
114             else:
115                 result = str(publish_date)[:4]
116 
117         return result or None
118 
119     def _get_subjects(self, data: dict[str, Any]) -> list[str] | None:
120         """Return a list of subjects None."""
121         subjects = [
122             subject.capitalize() for subject in data.get('subjects', '') if subject
123         ]
124         return subjects or None
125 
126     @staticmethod
127     def contributors(data: dict[str, Any]) -> list[dict[str, Any]] | None:
128         """Return a list of author-dicts or None."""
129 
130         def make_author(name):
131             author = {'name': name}
132             return author
133 
134         if contributors := data.get('authors'):
135             # form list of author dicts
136             authors = [make_author(c) for c in contributors if c[0]]
137             return authors
138 
139         return None
140 
141     def json(self):
142         """Return a JSON representation of the object."""
143         return {
144             field: getattr(self, field)
145             for field in self.ACTIVE_FIELDS
146             if getattr(self, field)
147         }
148 
149 
150 def load_state(path: str, logfile: str) -> tuple[list[str], int]:
151     """Retrieves starting point from logfile, if log exists
152 
153     Takes as input a path which expands to an ordered candidate list
154     of isbndb* filenames to process, the location of the
155     logfile, and determines which of those files are remaining, as
156     well as what our offset is in that file.
157 
158     e.g. if we request path containing f1, f2, f3 and our log
159     says f2,100 then we start our processing at f2 at the 100th line.
160 
161     This assumes the script is being called w/ e.g.:
162     /1/var/tmp/imports/2021-08/Bibliographic/*/
163     """
164     filenames = sorted(
165         os.path.join(path, f) for f in os.listdir(path) if f.startswith("isbndb")
166     )
167     try:
168         with open(logfile) as fin:
169             active_fname, offset = next(fin).strip().split(',')
170             unfinished_filenames = filenames[filenames.index(active_fname) :]
171             return unfinished_filenames, int(offset)
172     except (ValueError, OSError):
173         return filenames, 0
174 
175 
176 def get_line(line: bytes) -> dict | None:
177     """converts a line to a book item"""
178     json_object = None
179     try:
180         json_object = json.loads(line)
181     except JSONDecodeError as e:
182         logger.info(f"json decoding failed for: {line!r}: {e!r}")
183 
184     return json_object
185 
186 
187 def get_line_as_biblio(line: bytes, status: str) -> dict | None:
188     if json_object := get_line(line):
189         b = ISBNdb(json_object)
190         return {'ia_id': b.source_id, 'status': status, 'data': b.json()}
191 
192     return None
193 
194 
195 def update_state(logfile: str, fname: str, line_num: int = 0) -> None:
196     """Records the last file we began processing and the current line"""
197     with open(logfile, 'w') as fout:
198         fout.write(f'{fname},{line_num}\n')
199 
200 
201 # TODO: It's possible `batch_import()` could be modified to take a parsing function
202 # and a filter function instead of hardcoding in `csv_to_ol_json_item()` and some filters.
203 def batch_import(path: str, batch: Batch, import_status: str, batch_size: int = 5000):
204     logfile = os.path.join(path, 'import.log')
205     filenames, offset = load_state(path, logfile)
206 
207     for fname in filenames:
208         book_items = []
209         with open(fname, 'rb') as f:
210             logger.info(f"Processing: {fname} from line {offset}")
211             for line_num, line in enumerate(f):
212                 # skip over already processed records
213                 if offset:
214                     if offset > line_num:
215                         continue
216                     offset = 0
217 
218                 try:
219                     book_item = get_line_as_biblio(line=line, status=import_status)
220                     assert book_item is not None
221                     if not any(
222                         [
223                             "independently published"
224                             in book_item['data'].get('publishers', ''),
225                             is_published_in_future_year(book_item["data"]),
226                         ]
227                     ):
228                         book_items.append(book_item)
229                 except (AssertionError, IndexError) as e:
230                     logger.info(f"Error: {e!r} from {line!r}")
231 
232                 # If we have enough items, submit a batch
233                 if not ((line_num + 1) % batch_size):
234                     batch.add_items(book_items)
235                     update_state(logfile, fname, line_num)
236                     book_items = []  # clear added items
237 
238             # Add any remaining book_items to batch
239             if book_items:
240                 batch.add_items(book_items)
241             update_state(logfile, fname, line_num)
242 
243 
244 def main(ol_config: str, batch_path: str, import_status: str = 'staged') -> None:
245     load_config(ol_config)
246 
247     # Partner data is offset ~15 days from start of month
248     batch_name = "isbndb_bulk_import"
249     batch = Batch.find(batch_name) or Batch.new(batch_name)
250     batch_import(path=batch_path, batch=batch, import_status=import_status)
251 
252 
253 if __name__ == '__main__':
254     FnToCLI(main).run()
255
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scripts/providers/isbndb.py b/scripts/providers/isbndb.py
--- a/scripts/providers/isbndb.py
+++ b/scripts/providers/isbndb.py
@@ -71,7 +71,7 @@
         self.languages = self._get_languages(data)
         self.source_records = [self.source_id]
         self.subjects = [
-            subject.capitalize() for subject in data.get('subjects', '') if subject
+            subject.capitalize() for subject in (data.get('subjects') or []) if subject
         ]
         self.binding = data.get('binding', '')

{"golden_diff": "diff --git a/scripts/providers/isbndb.py b/scripts/providers/isbndb.py\n--- a/scripts/providers/isbndb.py\n+++ b/scripts/providers/isbndb.py\n@@ -71,7 +71,7 @@\n         self.languages = self._get_languages(data)\n         self.source_records = [self.source_id]\n         self.subjects = [\n-            subject.capitalize() for subject in data.get('subjects', '') if subject\n+            subject.capitalize() for subject in (data.get('subjects') or []) if subject\n         ]\n         self.binding = data.get('binding', '')\n", "issue": "`providers/isbndb.py` crashes when the `'subjects'` value is a literal `None`\n<!-- IMPORTANT: Before posting, be sure to redact or remove sensitive data, such as passwords, secret keys, session cookies, etc. -->\r\n<!-- What problem are we solving? What does the experience look like today? What are the symptoms? -->\r\n\r\n### Evidence / Screenshot (if possible)\r\n\r\n    \r\nSometimes the `'subjects'` key in the ISBNdb dump have literal `None` values, which would cause an error even if doing something like `data.get('subjects', [])`, because it would return `None` instead of `[]`.\r\n    \r\nE.g., consider the following record:\r\n```python\r\n{\r\n    'isbn': '0134878868',\r\n    'msrp': '0.00',\r\n    'title': 'Learn Adobe Premiere Pro Cc For Video Communication',\r\n    'isbn13': '9780134878867',\r\n    'authors': ['Dockery, Joe (author.)'],\r\n    'binding': 'Electronic Resource',\r\n    'language': 'en',\r\n    'subjects': None,\r\n}\r\n```\r\nThis would crash with:\r\n```python\r\n        ^^^^^^^^^^^^^^^^^^^\r\n  File \"/openlibrary/scripts/providers/isbndb.py\", line 73, in __init__\r\n    self.subjects = [\r\n                    ^\r\nTypeError: 'NoneType' object is not iterable\r\n```\r\n\r\n### Stakeholders\r\n<!-- @ tag stakeholders of this bug -->\r\n@mekarpeles \n", "before_files": [{"content": "import re\nimport json\nimport logging\nimport os\nfrom typing import Any, Final\nimport requests\n\nfrom json import JSONDecodeError\n\nfrom openlibrary.config import load_config\nfrom openlibrary.core.imports import Batch\nfrom openlibrary.plugins.upstream.utils import get_marc21_language\nfrom scripts.partner_batch_imports import is_published_in_future_year\nfrom scripts.solr_builder.solr_builder.fn_to_cli import FnToCLI\n\nlogger = logging.getLogger(\"openlibrary.importer.isbndb\")\n\nSCHEMA_URL = (\n    \"https://raw.githubusercontent.com/internetarchive\"\n    \"/openlibrary-client/master/olclient/schemata/import.schema.json\"\n)\n\nNONBOOK: Final = ['dvd', 'dvd-rom', 'cd', 'cd-rom', 'cassette', 'sheet music', 'audio']\nRE_YEAR = re.compile(r'(\\d{4})')\n\n\ndef is_nonbook(binding: str, nonbooks: list[str]) -> bool:\n    \"\"\"\n    Determine whether binding, or a substring of binding, split on \" \", is\n    contained within nonbooks.\n    \"\"\"\n    words = binding.split(\" \")\n    return any(word.casefold() in nonbooks for word in words)\n\n\nclass ISBNdb:\n    ACTIVE_FIELDS = [\n        'authors',\n        'isbn_13',\n        'languages',\n        'number_of_pages',\n        'publish_date',\n        'publishers',\n        'source_records',\n        'subjects',\n        'title',\n    ]\n    INACTIVE_FIELDS = [\n        \"copyright\",\n        \"dewey\",\n        \"doi\",\n        \"height\",\n        \"issn\",\n        \"lccn\",\n        \"length\",\n        \"width\",\n        'lc_classifications',\n        'pagination',\n        'weight',\n    ]\n    REQUIRED_FIELDS = requests.get(SCHEMA_URL).json()['required']\n\n    def __init__(self, data: dict[str, Any]):\n        self.isbn_13 = [data.get('isbn13')]\n        self.source_id = f'idb:{self.isbn_13[0]}'\n        self.title = data.get('title')\n        self.publish_date = self._get_year(data)  # 'YYYY'\n        self.publishers = self._get_list_if_present(data.get('publisher'))\n        self.authors = self.contributors(data)\n        self.number_of_pages = data.get('pages')\n        self.languages = self._get_languages(data)\n        self.source_records = [self.source_id]\n        self.subjects = [\n            subject.capitalize() for subject in data.get('subjects', '') if subject\n        ]\n        self.binding = data.get('binding', '')\n\n        # Assert importable\n        for field in self.REQUIRED_FIELDS + ['isbn_13']:\n            assert getattr(self, field), field\n        assert is_nonbook(self.binding, NONBOOK) is False, \"is_nonbook() returned True\"\n        assert self.isbn_13 != [\n            \"9780000000002\"\n        ], f\"known bad ISBN: {self.isbn_13}\"  # TODO: this should do more than ignore one known-bad ISBN.\n\n    def _get_languages(self, data: dict[str, Any]) -> list[str] | None:\n        \"\"\"Extract a list of MARC 21 format languages from an ISBNDb JSONL line.\"\"\"\n        language_line = data.get('language')\n        if not language_line:\n            return None\n\n        possible_languages = re.split(',| |;', language_line)\n        unique_languages = []\n\n        for language in possible_languages:\n            if (\n                marc21_language := get_marc21_language(language)\n            ) and marc21_language not in unique_languages:\n                unique_languages.append(marc21_language)\n\n        return unique_languages or None\n\n    def _get_list_if_present(self, item: str | None) -> list[str] | None:\n        \"\"\"Return items as a list, or None.\"\"\"\n        return [item] if item else None\n\n    def _get_year(self, data: dict[str, Any]) -> str | None:\n        \"\"\"Return a year str/int as a four digit string, or None.\"\"\"\n        result = \"\"\n        if publish_date := data.get('date_published'):\n            if isinstance(publish_date, str):\n                m = RE_YEAR.search(publish_date)\n                result = m.group(1) if m else None  # type: ignore[assignment]\n            else:\n                result = str(publish_date)[:4]\n\n        return result or None\n\n    def _get_subjects(self, data: dict[str, Any]) -> list[str] | None:\n        \"\"\"Return a list of subjects None.\"\"\"\n        subjects = [\n            subject.capitalize() for subject in data.get('subjects', '') if subject\n        ]\n        return subjects or None\n\n    @staticmethod\n    def contributors(data: dict[str, Any]) -> list[dict[str, Any]] | None:\n        \"\"\"Return a list of author-dicts or None.\"\"\"\n\n        def make_author(name):\n            author = {'name': name}\n            return author\n\n        if contributors := data.get('authors'):\n            # form list of author dicts\n            authors = [make_author(c) for c in contributors if c[0]]\n            return authors\n\n        return None\n\n    def json(self):\n        \"\"\"Return a JSON representation of the object.\"\"\"\n        return {\n            field: getattr(self, field)\n            for field in self.ACTIVE_FIELDS\n            if getattr(self, field)\n        }\n\n\ndef load_state(path: str, logfile: str) -> tuple[list[str], int]:\n    \"\"\"Retrieves starting point from logfile, if log exists\n\n    Takes as input a path which expands to an ordered candidate list\n    of isbndb* filenames to process, the location of the\n    logfile, and determines which of those files are remaining, as\n    well as what our offset is in that file.\n\n    e.g. if we request path containing f1, f2, f3 and our log\n    says f2,100 then we start our processing at f2 at the 100th line.\n\n    This assumes the script is being called w/ e.g.:\n    /1/var/tmp/imports/2021-08/Bibliographic/*/\n    \"\"\"\n    filenames = sorted(\n        os.path.join(path, f) for f in os.listdir(path) if f.startswith(\"isbndb\")\n    )\n    try:\n        with open(logfile) as fin:\n            active_fname, offset = next(fin).strip().split(',')\n            unfinished_filenames = filenames[filenames.index(active_fname) :]\n            return unfinished_filenames, int(offset)\n    except (ValueError, OSError):\n        return filenames, 0\n\n\ndef get_line(line: bytes) -> dict | None:\n    \"\"\"converts a line to a book item\"\"\"\n    json_object = None\n    try:\n        json_object = json.loads(line)\n    except JSONDecodeError as e:\n        logger.info(f\"json decoding failed for: {line!r}: {e!r}\")\n\n    return json_object\n\n\ndef get_line_as_biblio(line: bytes, status: str) -> dict | None:\n    if json_object := get_line(line):\n        b = ISBNdb(json_object)\n        return {'ia_id': b.source_id, 'status': status, 'data': b.json()}\n\n    return None\n\n\ndef update_state(logfile: str, fname: str, line_num: int = 0) -> None:\n    \"\"\"Records the last file we began processing and the current line\"\"\"\n    with open(logfile, 'w') as fout:\n        fout.write(f'{fname},{line_num}\\n')\n\n\n# TODO: It's possible `batch_import()` could be modified to take a parsing function\n# and a filter function instead of hardcoding in `csv_to_ol_json_item()` and some filters.\ndef batch_import(path: str, batch: Batch, import_status: str, batch_size: int = 5000):\n    logfile = os.path.join(path, 'import.log')\n    filenames, offset = load_state(path, logfile)\n\n    for fname in filenames:\n        book_items = []\n        with open(fname, 'rb') as f:\n            logger.info(f\"Processing: {fname} from line {offset}\")\n            for line_num, line in enumerate(f):\n                # skip over already processed records\n                if offset:\n                    if offset > line_num:\n                        continue\n                    offset = 0\n\n                try:\n                    book_item = get_line_as_biblio(line=line, status=import_status)\n                    assert book_item is not None\n                    if not any(\n                        [\n                            \"independently published\"\n                            in book_item['data'].get('publishers', ''),\n                            is_published_in_future_year(book_item[\"data\"]),\n                        ]\n                    ):\n                        book_items.append(book_item)\n                except (AssertionError, IndexError) as e:\n                    logger.info(f\"Error: {e!r} from {line!r}\")\n\n                # If we have enough items, submit a batch\n                if not ((line_num + 1) % batch_size):\n                    batch.add_items(book_items)\n                    update_state(logfile, fname, line_num)\n                    book_items = []  # clear added items\n\n            # Add any remaining book_items to batch\n            if book_items:\n                batch.add_items(book_items)\n            update_state(logfile, fname, line_num)\n\n\ndef main(ol_config: str, batch_path: str, import_status: str = 'staged') -> None:\n    load_config(ol_config)\n\n    # Partner data is offset ~15 days from start of month\n    batch_name = \"isbndb_bulk_import\"\n    batch = Batch.find(batch_name) or Batch.new(batch_name)\n    batch_import(path=batch_path, batch=batch, import_status=import_status)\n\n\nif __name__ == '__main__':\n    FnToCLI(main).run()\n", "path": "scripts/providers/isbndb.py"}], "after_files": [{"content": "import re\nimport json\nimport logging\nimport os\nfrom typing import Any, Final\nimport requests\n\nfrom json import JSONDecodeError\n\nfrom openlibrary.config import load_config\nfrom openlibrary.core.imports import Batch\nfrom openlibrary.plugins.upstream.utils import get_marc21_language\nfrom scripts.partner_batch_imports import is_published_in_future_year\nfrom scripts.solr_builder.solr_builder.fn_to_cli import FnToCLI\n\nlogger = logging.getLogger(\"openlibrary.importer.isbndb\")\n\nSCHEMA_URL = (\n    \"https://raw.githubusercontent.com/internetarchive\"\n    \"/openlibrary-client/master/olclient/schemata/import.schema.json\"\n)\n\nNONBOOK: Final = ['dvd', 'dvd-rom', 'cd', 'cd-rom', 'cassette', 'sheet music', 'audio']\nRE_YEAR = re.compile(r'(\\d{4})')\n\n\ndef is_nonbook(binding: str, nonbooks: list[str]) -> bool:\n    \"\"\"\n    Determine whether binding, or a substring of binding, split on \" \", is\n    contained within nonbooks.\n    \"\"\"\n    words = binding.split(\" \")\n    return any(word.casefold() in nonbooks for word in words)\n\n\nclass ISBNdb:\n    ACTIVE_FIELDS = [\n        'authors',\n        'isbn_13',\n        'languages',\n        'number_of_pages',\n        'publish_date',\n        'publishers',\n        'source_records',\n        'subjects',\n        'title',\n    ]\n    INACTIVE_FIELDS = [\n        \"copyright\",\n        \"dewey\",\n        \"doi\",\n        \"height\",\n        \"issn\",\n        \"lccn\",\n        \"length\",\n        \"width\",\n        'lc_classifications',\n        'pagination',\n        'weight',\n    ]\n    REQUIRED_FIELDS = requests.get(SCHEMA_URL).json()['required']\n\n    def __init__(self, data: dict[str, Any]):\n        self.isbn_13 = [data.get('isbn13')]\n        self.source_id = f'idb:{self.isbn_13[0]}'\n        self.title = data.get('title')\n        self.publish_date = self._get_year(data)  # 'YYYY'\n        self.publishers = self._get_list_if_present(data.get('publisher'))\n        self.authors = self.contributors(data)\n        self.number_of_pages = data.get('pages')\n        self.languages = self._get_languages(data)\n        self.source_records = [self.source_id]\n        self.subjects = [\n            subject.capitalize() for subject in (data.get('subjects') or []) if subject\n        ]\n        self.binding = data.get('binding', '')\n\n        # Assert importable\n        for field in self.REQUIRED_FIELDS + ['isbn_13']:\n            assert getattr(self, field), field\n        assert is_nonbook(self.binding, NONBOOK) is False, \"is_nonbook() returned True\"\n        assert self.isbn_13 != [\n            \"9780000000002\"\n        ], f\"known bad ISBN: {self.isbn_13}\"  # TODO: this should do more than ignore one known-bad ISBN.\n\n    def _get_languages(self, data: dict[str, Any]) -> list[str] | None:\n        \"\"\"Extract a list of MARC 21 format languages from an ISBNDb JSONL line.\"\"\"\n        language_line = data.get('language')\n        if not language_line:\n            return None\n\n        possible_languages = re.split(',| |;', language_line)\n        unique_languages = []\n\n        for language in possible_languages:\n            if (\n                marc21_language := get_marc21_language(language)\n            ) and marc21_language not in unique_languages:\n                unique_languages.append(marc21_language)\n\n        return unique_languages or None\n\n    def _get_list_if_present(self, item: str | None) -> list[str] | None:\n        \"\"\"Return items as a list, or None.\"\"\"\n        return [item] if item else None\n\n    def _get_year(self, data: dict[str, Any]) -> str | None:\n        \"\"\"Return a year str/int as a four digit string, or None.\"\"\"\n        result = \"\"\n        if publish_date := data.get('date_published'):\n            if isinstance(publish_date, str):\n                m = RE_YEAR.search(publish_date)\n                result = m.group(1) if m else None  # type: ignore[assignment]\n            else:\n                result = str(publish_date)[:4]\n\n        return result or None\n\n    def _get_subjects(self, data: dict[str, Any]) -> list[str] | None:\n        \"\"\"Return a list of subjects None.\"\"\"\n        subjects = [\n            subject.capitalize() for subject in data.get('subjects', '') if subject\n        ]\n        return subjects or None\n\n    @staticmethod\n    def contributors(data: dict[str, Any]) -> list[dict[str, Any]] | None:\n        \"\"\"Return a list of author-dicts or None.\"\"\"\n\n        def make_author(name):\n            author = {'name': name}\n            return author\n\n        if contributors := data.get('authors'):\n            # form list of author dicts\n            authors = [make_author(c) for c in contributors if c[0]]\n            return authors\n\n        return None\n\n    def json(self):\n        \"\"\"Return a JSON representation of the object.\"\"\"\n        return {\n            field: getattr(self, field)\n            for field in self.ACTIVE_FIELDS\n            if getattr(self, field)\n        }\n\n\ndef load_state(path: str, logfile: str) -> tuple[list[str], int]:\n    \"\"\"Retrieves starting point from logfile, if log exists\n\n    Takes as input a path which expands to an ordered candidate list\n    of isbndb* filenames to process, the location of the\n    logfile, and determines which of those files are remaining, as\n    well as what our offset is in that file.\n\n    e.g. if we request path containing f1, f2, f3 and our log\n    says f2,100 then we start our processing at f2 at the 100th line.\n\n    This assumes the script is being called w/ e.g.:\n    /1/var/tmp/imports/2021-08/Bibliographic/*/\n    \"\"\"\n    filenames = sorted(\n        os.path.join(path, f) for f in os.listdir(path) if f.startswith(\"isbndb\")\n    )\n    try:\n        with open(logfile) as fin:\n            active_fname, offset = next(fin).strip().split(',')\n            unfinished_filenames = filenames[filenames.index(active_fname) :]\n            return unfinished_filenames, int(offset)\n    except (ValueError, OSError):\n        return filenames, 0\n\n\ndef get_line(line: bytes) -> dict | None:\n    \"\"\"converts a line to a book item\"\"\"\n    json_object = None\n    try:\n        json_object = json.loads(line)\n    except JSONDecodeError as e:\n        logger.info(f\"json decoding failed for: {line!r}: {e!r}\")\n\n    return json_object\n\n\ndef get_line_as_biblio(line: bytes, status: str) -> dict | None:\n    if json_object := get_line(line):\n        b = ISBNdb(json_object)\n        return {'ia_id': b.source_id, 'status': status, 'data': b.json()}\n\n    return None\n\n\ndef update_state(logfile: str, fname: str, line_num: int = 0) -> None:\n    \"\"\"Records the last file we began processing and the current line\"\"\"\n    with open(logfile, 'w') as fout:\n        fout.write(f'{fname},{line_num}\\n')\n\n\n# TODO: It's possible `batch_import()` could be modified to take a parsing function\n# and a filter function instead of hardcoding in `csv_to_ol_json_item()` and some filters.\ndef batch_import(path: str, batch: Batch, import_status: str, batch_size: int = 5000):\n    logfile = os.path.join(path, 'import.log')\n    filenames, offset = load_state(path, logfile)\n\n    for fname in filenames:\n        book_items = []\n        with open(fname, 'rb') as f:\n            logger.info(f\"Processing: {fname} from line {offset}\")\n            for line_num, line in enumerate(f):\n                # skip over already processed records\n                if offset:\n                    if offset > line_num:\n                        continue\n                    offset = 0\n\n                try:\n                    book_item = get_line_as_biblio(line=line, status=import_status)\n                    assert book_item is not None\n                    if not any(\n                        [\n                            \"independently published\"\n                            in book_item['data'].get('publishers', ''),\n                            is_published_in_future_year(book_item[\"data\"]),\n                        ]\n                    ):\n                        book_items.append(book_item)\n                except (AssertionError, IndexError) as e:\n                    logger.info(f\"Error: {e!r} from {line!r}\")\n\n                # If we have enough items, submit a batch\n                if not ((line_num + 1) % batch_size):\n                    batch.add_items(book_items)\n                    update_state(logfile, fname, line_num)\n                    book_items = []  # clear added items\n\n            # Add any remaining book_items to batch\n            if book_items:\n                batch.add_items(book_items)\n            update_state(logfile, fname, line_num)\n\n\ndef main(ol_config: str, batch_path: str, import_status: str = 'staged') -> None:\n    load_config(ol_config)\n\n    # Partner data is offset ~15 days from start of month\n    batch_name = \"isbndb_bulk_import\"\n    batch = Batch.find(batch_name) or Batch.new(batch_name)\n    batch_import(path=batch_path, batch=batch, import_status=import_status)\n\n\nif __name__ == '__main__':\n    FnToCLI(main).run()\n", "path": "scripts/providers/isbndb.py"}]}