We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
solrupdater getting randomly stuck
Solrupdater has been getting stuck randomly at various dates. It seems related to the infobase py3 deploy.

### Steps to Reproduce
<!-- What steps caused you to find the bug? -->
1. Do nothing

<!-- What actually happened after these steps? What did you expect to happen? -->
* Actual: It seems to occasionally get stuck in the log
* Expected: Solrupdater runs through the infobase logs, making updates

### Proposal & Constraints
<!-- What is the proposed solution / implementation? Is there a precedent of this approach succeeding elsewhere? -->

### Related files
<!-- Files related to this issue; this is super useful for new contributors who might want to help! If you're not sure, leave this blank; a maintainer will add them. -->

### Stakeholders
@seabelis 


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scripts/new-solr-updater.py`
Content:
```
1 """New script to handle solr updates.
2 
3 Author: Anand Chitipothu
4 
5 Changes:
6 2013-02-25: First version
7 2018-02-11: Use newer config method
8 """
9 import _init_path
10 
11 from six.moves import urllib
12 import yaml
13 import logging
14 import json
15 import argparse
16 import datetime
17 import time
18 import web
19 import sys
20 import re
21 import socket
22 
23 from openlibrary.solr import update_work
24 from openlibrary.config import load_config
25 from infogami import config
26 
27 logger = logging.getLogger("openlibrary.solr-updater")
28 
29 LOAD_IA_SCANS = False
30 COMMIT = True
31 args = {}
32 
33 
34 def parse_arguments():
35     parser = argparse.ArgumentParser()
36     parser.add_argument('-c', '--config')
37     parser.add_argument('--debugger', action="store_true", help="Wait for a debugger to attach before beginning.")
38     parser.add_argument('--state-file', default="solr-update.state")
39     parser.add_argument('--exclude-edits-containing', help="Don't index matching edits")
40     parser.add_argument('--ol-url', default="http://openlibrary.org/")
41     parser.add_argument('--socket-timeout', type=int, default=10)
42     parser.add_argument('--load-ia-scans', dest="load_ia_scans", action="store_true", default=False)
43     parser.add_argument('--no-commit', dest="commit", action="store_false", default=True)
44     return parser.parse_args()
45 
46 def read_state_file(path):
47     try:
48         return open(path).read()
49     except IOError:
50         logger.error("State file %s is not found. Reading log from the beginning of today", path)
51         return datetime.date.today().isoformat() + ":0"
52 
53 def get_default_offset():
54     return datetime.date.today().isoformat() + ":0"
55 
56 
57 class InfobaseLog:
58     def __init__(self, hostname, exclude=None):
59         """
60         :param str hostname:
61         :param str|None exclude: if specified, excludes records that include the string
62         """
63         self.base_url = 'http://%s/openlibrary.org/log' % hostname
64         self.offset = get_default_offset()
65         self.exclude = exclude
66 
67     def tell(self):
68         return self.offset
69 
70     def seek(self, offset):
71         self.offset = offset.strip()
72 
73     def read_records(self, max_fetches=10):
74         """Reads all the available log records from the server.
75         """
76         for i in range(max_fetches):
77             url = "%s/%s?limit=100" % (self.base_url, self.offset)
78             logger.debug("Reading log from %s", url)
79             try:
80                 jsontext = urllib.request.urlopen(url).read()
81             except urllib.error.URLError as e:
82                 logger.error("Failed to open URL %s", url, exc_info=True)
83                 if e.args and e.args[0].args == (111, 'Connection refused'):
84                     logger.error('make sure infogami server is working, connection refused from %s', url)
85                     sys.exit(1)
86                 raise
87 
88             try:
89                 d = json.loads(jsontext)
90             except:
91                 logger.error("Bad JSON: %s", jsontext)
92                 raise
93             data = d['data']
94             # no more data is available
95             if not data:
96                 logger.debug("no more records found")
97                 return
98 
99             for record in data:
100                 if self.exclude and self.exclude in json.dumps(record):
101                     continue
102                 yield record
103 
104             self.offset = d['offset']
105 
106 def parse_log(records):
107     for rec in records:
108         action = rec.get('action')
109         if action == 'save':
110             key = rec['data'].get('key')
111             if key:
112                 yield key
113         elif action == 'save_many':
114             changes = rec['data'].get('changeset', {}).get('changes', [])
115             for c in changes:
116                 yield c['key']
117 
118         elif action == 'store.put':
119             # A sample record looks like this:
120             # {
121             #   "action": "store.put",
122             #   "timestamp": "2011-12-01T00:00:44.241604",
123             #   "data": {
124             #       "data": {"borrowed": "false", "_key": "ebooks/books/OL5854888M", "_rev": "975708", "type": "ebook", "book_key": "/books/OL5854888M"},
125             #       "key": "ebooks/books/OL5854888M"
126             #   },
127             #   "site": "openlibrary.org"
128             # }
129             data = rec.get('data', {}).get("data", {})
130             key = data.get("_key", "")
131             if data.get("type") == "ebook" and key.startswith("ebooks/books/"):
132                 edition_key = data.get('book_key')
133                 if edition_key:
134                     yield edition_key
135             elif LOAD_IA_SCANS and data.get("type") == "ia-scan" and key.startswith("ia-scan/"):
136                 identifier = data.get('identifier')
137                 if identifier and is_allowed_itemid(identifier):
138                     yield "/books/ia:" + identifier
139 
140             # Hack to force updating something from admin interface
141             # The admin interface writes the keys to update to a document named
142             # 'solr-force-update' in the store and whatever keys are written to that
143             # are picked by this script
144             elif key == 'solr-force-update':
145                 keys = data.get('keys')
146                 for k in keys:
147                     yield k
148 
149         elif action == 'store.delete':
150             key = rec.get("data", {}).get("key")
151             # An ia-scan key is deleted when that book is deleted/darked from IA.
152             # Delete it from OL solr by updating that key
153             if key.startswith("ia-scan/"):
154                 ol_key = "/works/ia:" + key.split("/")[-1]
155                 yield ol_key
156 
157 def is_allowed_itemid(identifier):
158     if not re.match("^[a-zA-Z0-9_.-]*$", identifier):
159         return False
160 
161     # items starts with these prefixes are not books. Ignore them.
162     ignore_prefixes = config.get("ia_ignore_prefixes", [])
163     for prefix in ignore_prefixes:
164         if identifier.startswith(prefix):
165             return False
166 
167     return True
168 
169 def update_keys(keys):
170     if not keys:
171         return 0
172 
173     # FIXME: Some kind of hack introduced to work around DB connectivity issue
174     global args
175     logger.debug("Args: %s" % str(args))
176     update_work.load_configs(args.ol_url, args.config, 'default')
177 
178     keys = [k for k in keys if k.count("/") == 2 and k.split("/")[1] in ("books", "authors", "works")]
179 
180     count = 0
181     for chunk in web.group(keys, 100):
182         chunk = list(chunk)
183         count += len(chunk)
184         update_work.do_updates(chunk)
185 
186     if count:
187         logger.info("updated %d documents", count)
188 
189     return count
190 
191 class Solr:
192     def __init__(self):
193         self.reset()
194 
195     def reset(self):
196         self.total_docs = 0
197         self.t_start = time.time()
198 
199     def commit(self, ndocs):
200         """Performs solr commit only if there are sufficient number
201         of documents or enough time has been passed since last commit.
202         """
203         self.total_docs += ndocs
204 
205         # no documents to commit
206         if not self.total_docs:
207             return
208 
209         dt = time.time() - self.t_start
210         if self.total_docs > 100 or dt > 60:
211             logger.info("doing solr commit (%d docs updated, last commit was %0.1f seconds ago)", self.total_docs, dt)
212             self._solr_commit()
213             self.reset()
214         else:
215             logger.debug("skipping solr commit (%d docs updated, last commit was %0.1f seconds ago)", self.total_docs, dt)
216 
217     def _solr_commit(self):
218         logger.info("BEGIN commit")
219         update_work.solr_update(['<commit/>'])
220         logger.info("END commit")
221 
222 
223 def process_args(args):
224     if args.debugger:
225         import ptvsd
226 
227         logger.info("Enabling debugger attachment (attach if it hangs here)")
228         ptvsd.enable_attach(address=('0.0.0.0', 3000))
229         logger.info("Waiting for debugger to attach...")
230         ptvsd.wait_for_attach()
231         logger.info("Debugger attached to port 3000")
232 
233     # Sometimes archive.org requests blocks forever.
234     # Setting a timeout will make the request fail instead of waiting forever.
235     socket.setdefaulttimeout(args.socket_timeout)
236 
237     global LOAD_IA_SCANS, COMMIT
238     LOAD_IA_SCANS = args.load_ia_scans
239     COMMIT = args.commit
240 
241 
242 def main():
243     global args
244     FORMAT = "%(asctime)-15s %(levelname)s %(message)s"
245     logging.basicConfig(level=logging.INFO, format=FORMAT)
246 
247     logger.info("BEGIN new-solr-updater")
248 
249     args = parse_arguments()
250     process_args(args)
251 
252     # set OL URL when running on a dev-instance
253     if args.ol_url:
254         host = web.lstrips(args.ol_url, "http://").strip("/")
255         update_work.set_query_host(host)
256 
257     logger.info(str(args))
258     logger.info("loading config from %s", args.config)
259     load_config(args.config)
260 
261     state_file = args.state_file
262     offset = read_state_file(state_file)
263 
264     logfile = InfobaseLog(config.get('infobase_server'),
265                           exclude=args.exclude_edits_containing)
266     logfile.seek(offset)
267 
268     solr = Solr()
269 
270     while True:
271         records = logfile.read_records()
272         keys = parse_log(records)
273         count = update_keys(keys)
274 
275         if logfile.tell() != offset:
276             offset = logfile.tell()
277             logger.info("saving offset %s", offset)
278             with open(state_file, "w") as f:
279                 f.write(offset)
280 
281         if COMMIT:
282             solr.commit(ndocs=count)
283         else:
284             logger.info("not doing solr commit as commit is off")
285 
286         # don't sleep after committing some records.
287         # While the commit was on, some more edits might have happened.
288         if count == 0:
289             logger.debug("No more log records available, sleeping...")
290             time.sleep(5)
291 
292 
293 if __name__ == "__main__":
294     main()
295
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scripts/new-solr-updater.py b/scripts/new-solr-updater.py
--- a/scripts/new-solr-updater.py
+++ b/scripts/new-solr-updater.py
@@ -94,6 +94,18 @@
             # no more data is available
             if not data:
                 logger.debug("no more records found")
+                # There's an infobase bug where we'll sometimes get 0 items, but the
+                # binary offset will have incremented...?
+                if 'offset' in d:
+                    # There's _another_ infobase bug where if you query a future date,
+                    # it'll return back 2020-12-01. To avoid solrupdater getting stuck
+                    # in a loop, only update the offset if it's newer than the current
+                    old_day, old_boffset = self.offset.split(':')
+                    old_boffset = int(old_boffset)
+                    new_day, new_boffset = d['offset'].split(':')
+                    new_boffset = int(new_boffset)
+                    if new_day >= old_day and new_boffset >= old_boffset:
+                        self.offset = d['offset']
                 return
 
             for record in data:

{"golden_diff": "diff --git a/scripts/new-solr-updater.py b/scripts/new-solr-updater.py\n--- a/scripts/new-solr-updater.py\n+++ b/scripts/new-solr-updater.py\n@@ -94,6 +94,18 @@\n             # no more data is available\n             if not data:\n                 logger.debug(\"no more records found\")\n+                # There's an infobase bug where we'll sometimes get 0 items, but the\n+                # binary offset will have incremented...?\n+                if 'offset' in d:\n+                    # There's _another_ infobase bug where if you query a future date,\n+                    # it'll return back 2020-12-01. To avoid solrupdater getting stuck\n+                    # in a loop, only update the offset if it's newer than the current\n+                    old_day, old_boffset = self.offset.split(':')\n+                    old_boffset = int(old_boffset)\n+                    new_day, new_boffset = d['offset'].split(':')\n+                    new_boffset = int(new_boffset)\n+                    if new_day >= old_day and new_boffset >= old_boffset:\n+                        self.offset = d['offset']\n                 return\n \n             for record in data:\n", "issue": "solrupdater getting randomly stuck\nSolrupdater has been getting stuck randomly at various dates. It seems related to the infobase py3 deploy.\r\n\r\n### Steps to Reproduce\r\n<!-- What steps caused you to find the bug? -->\r\n1. Do nothing\r\n\r\n<!-- What actually happened after these steps? What did you expect to happen? -->\r\n* Actual: It seems to occasionally get stuck in the log\r\n* Expected: Solrupdater runs through the infobase logs, making updates\r\n\r\n### Proposal & Constraints\r\n<!-- What is the proposed solution / implementation? Is there a precedent of this approach succeeding elsewhere? -->\r\n\r\n### Related files\r\n<!-- Files related to this issue; this is super useful for new contributors who might want to help! If you're not sure, leave this blank; a maintainer will add them. -->\r\n\r\n### Stakeholders\r\n@seabelis \r\n\n", "before_files": [{"content": "\"\"\"New script to handle solr updates.\n\nAuthor: Anand Chitipothu\n\nChanges:\n2013-02-25: First version\n2018-02-11: Use newer config method\n\"\"\"\nimport _init_path\n\nfrom six.moves import urllib\nimport yaml\nimport logging\nimport json\nimport argparse\nimport datetime\nimport time\nimport web\nimport sys\nimport re\nimport socket\n\nfrom openlibrary.solr import update_work\nfrom openlibrary.config import load_config\nfrom infogami import config\n\nlogger = logging.getLogger(\"openlibrary.solr-updater\")\n\nLOAD_IA_SCANS = False\nCOMMIT = True\nargs = {}\n\n\ndef parse_arguments():\n    parser = argparse.ArgumentParser()\n    parser.add_argument('-c', '--config')\n    parser.add_argument('--debugger', action=\"store_true\", help=\"Wait for a debugger to attach before beginning.\")\n    parser.add_argument('--state-file', default=\"solr-update.state\")\n    parser.add_argument('--exclude-edits-containing', help=\"Don't index matching edits\")\n    parser.add_argument('--ol-url', default=\"http://openlibrary.org/\")\n    parser.add_argument('--socket-timeout', type=int, default=10)\n    parser.add_argument('--load-ia-scans', dest=\"load_ia_scans\", action=\"store_true\", default=False)\n    parser.add_argument('--no-commit', dest=\"commit\", action=\"store_false\", default=True)\n    return parser.parse_args()\n\ndef read_state_file(path):\n    try:\n        return open(path).read()\n    except IOError:\n        logger.error(\"State file %s is not found. Reading log from the beginning of today\", path)\n        return datetime.date.today().isoformat() + \":0\"\n\ndef get_default_offset():\n    return datetime.date.today().isoformat() + \":0\"\n\n\nclass InfobaseLog:\n    def __init__(self, hostname, exclude=None):\n        \"\"\"\n        :param str hostname:\n        :param str|None exclude: if specified, excludes records that include the string\n        \"\"\"\n        self.base_url = 'http://%s/openlibrary.org/log' % hostname\n        self.offset = get_default_offset()\n        self.exclude = exclude\n\n    def tell(self):\n        return self.offset\n\n    def seek(self, offset):\n        self.offset = offset.strip()\n\n    def read_records(self, max_fetches=10):\n        \"\"\"Reads all the available log records from the server.\n        \"\"\"\n        for i in range(max_fetches):\n            url = \"%s/%s?limit=100\" % (self.base_url, self.offset)\n            logger.debug(\"Reading log from %s\", url)\n            try:\n                jsontext = urllib.request.urlopen(url).read()\n            except urllib.error.URLError as e:\n                logger.error(\"Failed to open URL %s\", url, exc_info=True)\n                if e.args and e.args[0].args == (111, 'Connection refused'):\n                    logger.error('make sure infogami server is working, connection refused from %s', url)\n                    sys.exit(1)\n                raise\n\n            try:\n                d = json.loads(jsontext)\n            except:\n                logger.error(\"Bad JSON: %s\", jsontext)\n                raise\n            data = d['data']\n            # no more data is available\n            if not data:\n                logger.debug(\"no more records found\")\n                return\n\n            for record in data:\n                if self.exclude and self.exclude in json.dumps(record):\n                    continue\n                yield record\n\n            self.offset = d['offset']\n\ndef parse_log(records):\n    for rec in records:\n        action = rec.get('action')\n        if action == 'save':\n            key = rec['data'].get('key')\n            if key:\n                yield key\n        elif action == 'save_many':\n            changes = rec['data'].get('changeset', {}).get('changes', [])\n            for c in changes:\n                yield c['key']\n\n        elif action == 'store.put':\n            # A sample record looks like this:\n            # {\n            #   \"action\": \"store.put\",\n            #   \"timestamp\": \"2011-12-01T00:00:44.241604\",\n            #   \"data\": {\n            #       \"data\": {\"borrowed\": \"false\", \"_key\": \"ebooks/books/OL5854888M\", \"_rev\": \"975708\", \"type\": \"ebook\", \"book_key\": \"/books/OL5854888M\"},\n            #       \"key\": \"ebooks/books/OL5854888M\"\n            #   },\n            #   \"site\": \"openlibrary.org\"\n            # }\n            data = rec.get('data', {}).get(\"data\", {})\n            key = data.get(\"_key\", \"\")\n            if data.get(\"type\") == \"ebook\" and key.startswith(\"ebooks/books/\"):\n                edition_key = data.get('book_key')\n                if edition_key:\n                    yield edition_key\n            elif LOAD_IA_SCANS and data.get(\"type\") == \"ia-scan\" and key.startswith(\"ia-scan/\"):\n                identifier = data.get('identifier')\n                if identifier and is_allowed_itemid(identifier):\n                    yield \"/books/ia:\" + identifier\n\n            # Hack to force updating something from admin interface\n            # The admin interface writes the keys to update to a document named\n            # 'solr-force-update' in the store and whatever keys are written to that\n            # are picked by this script\n            elif key == 'solr-force-update':\n                keys = data.get('keys')\n                for k in keys:\n                    yield k\n\n        elif action == 'store.delete':\n            key = rec.get(\"data\", {}).get(\"key\")\n            # An ia-scan key is deleted when that book is deleted/darked from IA.\n            # Delete it from OL solr by updating that key\n            if key.startswith(\"ia-scan/\"):\n                ol_key = \"/works/ia:\" + key.split(\"/\")[-1]\n                yield ol_key\n\ndef is_allowed_itemid(identifier):\n    if not re.match(\"^[a-zA-Z0-9_.-]*$\", identifier):\n        return False\n\n    # items starts with these prefixes are not books. Ignore them.\n    ignore_prefixes = config.get(\"ia_ignore_prefixes\", [])\n    for prefix in ignore_prefixes:\n        if identifier.startswith(prefix):\n            return False\n\n    return True\n\ndef update_keys(keys):\n    if not keys:\n        return 0\n\n    # FIXME: Some kind of hack introduced to work around DB connectivity issue\n    global args\n    logger.debug(\"Args: %s\" % str(args))\n    update_work.load_configs(args.ol_url, args.config, 'default')\n\n    keys = [k for k in keys if k.count(\"/\") == 2 and k.split(\"/\")[1] in (\"books\", \"authors\", \"works\")]\n\n    count = 0\n    for chunk in web.group(keys, 100):\n        chunk = list(chunk)\n        count += len(chunk)\n        update_work.do_updates(chunk)\n\n    if count:\n        logger.info(\"updated %d documents\", count)\n\n    return count\n\nclass Solr:\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.total_docs = 0\n        self.t_start = time.time()\n\n    def commit(self, ndocs):\n        \"\"\"Performs solr commit only if there are sufficient number\n        of documents or enough time has been passed since last commit.\n        \"\"\"\n        self.total_docs += ndocs\n\n        # no documents to commit\n        if not self.total_docs:\n            return\n\n        dt = time.time() - self.t_start\n        if self.total_docs > 100 or dt > 60:\n            logger.info(\"doing solr commit (%d docs updated, last commit was %0.1f seconds ago)\", self.total_docs, dt)\n            self._solr_commit()\n            self.reset()\n        else:\n            logger.debug(\"skipping solr commit (%d docs updated, last commit was %0.1f seconds ago)\", self.total_docs, dt)\n\n    def _solr_commit(self):\n        logger.info(\"BEGIN commit\")\n        update_work.solr_update(['<commit/>'])\n        logger.info(\"END commit\")\n\n\ndef process_args(args):\n    if args.debugger:\n        import ptvsd\n\n        logger.info(\"Enabling debugger attachment (attach if it hangs here)\")\n        ptvsd.enable_attach(address=('0.0.0.0', 3000))\n        logger.info(\"Waiting for debugger to attach...\")\n        ptvsd.wait_for_attach()\n        logger.info(\"Debugger attached to port 3000\")\n\n    # Sometimes archive.org requests blocks forever.\n    # Setting a timeout will make the request fail instead of waiting forever.\n    socket.setdefaulttimeout(args.socket_timeout)\n\n    global LOAD_IA_SCANS, COMMIT\n    LOAD_IA_SCANS = args.load_ia_scans\n    COMMIT = args.commit\n\n\ndef main():\n    global args\n    FORMAT = \"%(asctime)-15s %(levelname)s %(message)s\"\n    logging.basicConfig(level=logging.INFO, format=FORMAT)\n\n    logger.info(\"BEGIN new-solr-updater\")\n\n    args = parse_arguments()\n    process_args(args)\n\n    # set OL URL when running on a dev-instance\n    if args.ol_url:\n        host = web.lstrips(args.ol_url, \"http://\").strip(\"/\")\n        update_work.set_query_host(host)\n\n    logger.info(str(args))\n    logger.info(\"loading config from %s\", args.config)\n    load_config(args.config)\n\n    state_file = args.state_file\n    offset = read_state_file(state_file)\n\n    logfile = InfobaseLog(config.get('infobase_server'),\n                          exclude=args.exclude_edits_containing)\n    logfile.seek(offset)\n\n    solr = Solr()\n\n    while True:\n        records = logfile.read_records()\n        keys = parse_log(records)\n        count = update_keys(keys)\n\n        if logfile.tell() != offset:\n            offset = logfile.tell()\n            logger.info(\"saving offset %s\", offset)\n            with open(state_file, \"w\") as f:\n                f.write(offset)\n\n        if COMMIT:\n            solr.commit(ndocs=count)\n        else:\n            logger.info(\"not doing solr commit as commit is off\")\n\n        # don't sleep after committing some records.\n        # While the commit was on, some more edits might have happened.\n        if count == 0:\n            logger.debug(\"No more log records available, sleeping...\")\n            time.sleep(5)\n\n\nif __name__ == \"__main__\":\n    main()\n", "path": "scripts/new-solr-updater.py"}], "after_files": [{"content": "\"\"\"New script to handle solr updates.\n\nAuthor: Anand Chitipothu\n\nChanges:\n2013-02-25: First version\n2018-02-11: Use newer config method\n\"\"\"\nimport _init_path\n\nfrom six.moves import urllib\nimport yaml\nimport logging\nimport json\nimport argparse\nimport datetime\nimport time\nimport web\nimport sys\nimport re\nimport socket\n\nfrom openlibrary.solr import update_work\nfrom openlibrary.config import load_config\nfrom infogami import config\n\nlogger = logging.getLogger(\"openlibrary.solr-updater\")\n\nLOAD_IA_SCANS = False\nCOMMIT = True\nargs = {}\n\n\ndef parse_arguments():\n    parser = argparse.ArgumentParser()\n    parser.add_argument('-c', '--config')\n    parser.add_argument('--debugger', action=\"store_true\", help=\"Wait for a debugger to attach before beginning.\")\n    parser.add_argument('--state-file', default=\"solr-update.state\")\n    parser.add_argument('--exclude-edits-containing', help=\"Don't index matching edits\")\n    parser.add_argument('--ol-url', default=\"http://openlibrary.org/\")\n    parser.add_argument('--socket-timeout', type=int, default=10)\n    parser.add_argument('--load-ia-scans', dest=\"load_ia_scans\", action=\"store_true\", default=False)\n    parser.add_argument('--no-commit', dest=\"commit\", action=\"store_false\", default=True)\n    return parser.parse_args()\n\ndef read_state_file(path):\n    try:\n        return open(path).read()\n    except IOError:\n        logger.error(\"State file %s is not found. Reading log from the beginning of today\", path)\n        return datetime.date.today().isoformat() + \":0\"\n\ndef get_default_offset():\n    return datetime.date.today().isoformat() + \":0\"\n\n\nclass InfobaseLog:\n    def __init__(self, hostname, exclude=None):\n        \"\"\"\n        :param str hostname:\n        :param str|None exclude: if specified, excludes records that include the string\n        \"\"\"\n        self.base_url = 'http://%s/openlibrary.org/log' % hostname\n        self.offset = get_default_offset()\n        self.exclude = exclude\n\n    def tell(self):\n        return self.offset\n\n    def seek(self, offset):\n        self.offset = offset.strip()\n\n    def read_records(self, max_fetches=10):\n        \"\"\"Reads all the available log records from the server.\n        \"\"\"\n        for i in range(max_fetches):\n            url = \"%s/%s?limit=100\" % (self.base_url, self.offset)\n            logger.debug(\"Reading log from %s\", url)\n            try:\n                jsontext = urllib.request.urlopen(url).read()\n            except urllib.error.URLError as e:\n                logger.error(\"Failed to open URL %s\", url, exc_info=True)\n                if e.args and e.args[0].args == (111, 'Connection refused'):\n                    logger.error('make sure infogami server is working, connection refused from %s', url)\n                    sys.exit(1)\n                raise\n\n            try:\n                d = json.loads(jsontext)\n            except:\n                logger.error(\"Bad JSON: %s\", jsontext)\n                raise\n            data = d['data']\n            # no more data is available\n            if not data:\n                logger.debug(\"no more records found\")\n                # There's an infobase bug where we'll sometimes get 0 items, but the\n                # binary offset will have incremented...?\n                if 'offset' in d:\n                    # There's _another_ infobase bug where if you query a future date,\n                    # it'll return back 2020-12-01. To avoid solrupdater getting stuck\n                    # in a loop, only update the offset if it's newer than the current\n                    old_day, old_boffset = self.offset.split(':')\n                    old_boffset = int(old_boffset)\n                    new_day, new_boffset = d['offset'].split(':')\n                    new_boffset = int(new_boffset)\n                    if new_day >= old_day and new_boffset >= old_boffset:\n                        self.offset = d['offset']\n                return\n\n            for record in data:\n                if self.exclude and self.exclude in json.dumps(record):\n                    continue\n                yield record\n\n            self.offset = d['offset']\n\ndef parse_log(records):\n    for rec in records:\n        action = rec.get('action')\n        if action == 'save':\n            key = rec['data'].get('key')\n            if key:\n                yield key\n        elif action == 'save_many':\n            changes = rec['data'].get('changeset', {}).get('changes', [])\n            for c in changes:\n                yield c['key']\n\n        elif action == 'store.put':\n            # A sample record looks like this:\n            # {\n            #   \"action\": \"store.put\",\n            #   \"timestamp\": \"2011-12-01T00:00:44.241604\",\n            #   \"data\": {\n            #       \"data\": {\"borrowed\": \"false\", \"_key\": \"ebooks/books/OL5854888M\", \"_rev\": \"975708\", \"type\": \"ebook\", \"book_key\": \"/books/OL5854888M\"},\n            #       \"key\": \"ebooks/books/OL5854888M\"\n            #   },\n            #   \"site\": \"openlibrary.org\"\n            # }\n            data = rec.get('data', {}).get(\"data\", {})\n            key = data.get(\"_key\", \"\")\n            if data.get(\"type\") == \"ebook\" and key.startswith(\"ebooks/books/\"):\n                edition_key = data.get('book_key')\n                if edition_key:\n                    yield edition_key\n            elif LOAD_IA_SCANS and data.get(\"type\") == \"ia-scan\" and key.startswith(\"ia-scan/\"):\n                identifier = data.get('identifier')\n                if identifier and is_allowed_itemid(identifier):\n                    yield \"/books/ia:\" + identifier\n\n            # Hack to force updating something from admin interface\n            # The admin interface writes the keys to update to a document named\n            # 'solr-force-update' in the store and whatever keys are written to that\n            # are picked by this script\n            elif key == 'solr-force-update':\n                keys = data.get('keys')\n                for k in keys:\n                    yield k\n\n        elif action == 'store.delete':\n            key = rec.get(\"data\", {}).get(\"key\")\n            # An ia-scan key is deleted when that book is deleted/darked from IA.\n            # Delete it from OL solr by updating that key\n            if key.startswith(\"ia-scan/\"):\n                ol_key = \"/works/ia:\" + key.split(\"/\")[-1]\n                yield ol_key\n\ndef is_allowed_itemid(identifier):\n    if not re.match(\"^[a-zA-Z0-9_.-]*$\", identifier):\n        return False\n\n    # items starts with these prefixes are not books. Ignore them.\n    ignore_prefixes = config.get(\"ia_ignore_prefixes\", [])\n    for prefix in ignore_prefixes:\n        if identifier.startswith(prefix):\n            return False\n\n    return True\n\ndef update_keys(keys):\n    if not keys:\n        return 0\n\n    # FIXME: Some kind of hack introduced to work around DB connectivity issue\n    global args\n    logger.debug(\"Args: %s\" % str(args))\n    update_work.load_configs(args.ol_url, args.config, 'default')\n\n    keys = [k for k in keys if k.count(\"/\") == 2 and k.split(\"/\")[1] in (\"books\", \"authors\", \"works\")]\n\n    count = 0\n    for chunk in web.group(keys, 100):\n        chunk = list(chunk)\n        count += len(chunk)\n        update_work.do_updates(chunk)\n\n    if count:\n        logger.info(\"updated %d documents\", count)\n\n    return count\n\nclass Solr:\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.total_docs = 0\n        self.t_start = time.time()\n\n    def commit(self, ndocs):\n        \"\"\"Performs solr commit only if there are sufficient number\n        of documents or enough time has been passed since last commit.\n        \"\"\"\n        self.total_docs += ndocs\n\n        # no documents to commit\n        if not self.total_docs:\n            return\n\n        dt = time.time() - self.t_start\n        if self.total_docs > 100 or dt > 60:\n            logger.info(\"doing solr commit (%d docs updated, last commit was %0.1f seconds ago)\", self.total_docs, dt)\n            self._solr_commit()\n            self.reset()\n        else:\n            logger.debug(\"skipping solr commit (%d docs updated, last commit was %0.1f seconds ago)\", self.total_docs, dt)\n\n    def _solr_commit(self):\n        logger.info(\"BEGIN commit\")\n        update_work.solr_update(['<commit/>'])\n        logger.info(\"END commit\")\n\n\ndef process_args(args):\n    if args.debugger:\n        import ptvsd\n\n        logger.info(\"Enabling debugger attachment (attach if it hangs here)\")\n        ptvsd.enable_attach(address=('0.0.0.0', 3000))\n        logger.info(\"Waiting for debugger to attach...\")\n        ptvsd.wait_for_attach()\n        logger.info(\"Debugger attached to port 3000\")\n\n    # Sometimes archive.org requests blocks forever.\n    # Setting a timeout will make the request fail instead of waiting forever.\n    socket.setdefaulttimeout(args.socket_timeout)\n\n    global LOAD_IA_SCANS, COMMIT\n    LOAD_IA_SCANS = args.load_ia_scans\n    COMMIT = args.commit\n\n\ndef main():\n    global args\n    FORMAT = \"%(asctime)-15s %(levelname)s %(message)s\"\n    logging.basicConfig(level=logging.INFO, format=FORMAT)\n\n    logger.info(\"BEGIN new-solr-updater\")\n\n    args = parse_arguments()\n    process_args(args)\n\n    # set OL URL when running on a dev-instance\n    if args.ol_url:\n        host = web.lstrips(args.ol_url, \"http://\").strip(\"/\")\n        update_work.set_query_host(host)\n\n    logger.info(str(args))\n    logger.info(\"loading config from %s\", args.config)\n    load_config(args.config)\n\n    state_file = args.state_file\n    offset = read_state_file(state_file)\n\n    logfile = InfobaseLog(config.get('infobase_server'),\n                          exclude=args.exclude_edits_containing)\n    logfile.seek(offset)\n\n    solr = Solr()\n\n    while True:\n        records = logfile.read_records()\n        keys = parse_log(records)\n        count = update_keys(keys)\n\n        if logfile.tell() != offset:\n            offset = logfile.tell()\n            logger.info(\"saving offset %s\", offset)\n            with open(state_file, \"w\") as f:\n                f.write(offset)\n\n        if COMMIT:\n            solr.commit(ndocs=count)\n        else:\n            logger.info(\"not doing solr commit as commit is off\")\n\n        # don't sleep after committing some records.\n        # While the commit was on, some more edits might have happened.\n        if count == 0:\n            logger.debug(\"No more log records available, sleeping...\")\n            time.sleep(5)\n\n\nif __name__ == \"__main__\":\n    main()\n", "path": "scripts/new-solr-updater.py"}]}