We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Stop bad scripts from causing errors
Currently when `compile_json` gets run if something goes wrong the retriever errors out even on commands not running the script (e.g., `retriever ls`). What it should do is ignore the bad script and possibly report back that there is an issue with the script but keep running normally otherwise.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `lib/compile.py`
Content:
```
1 from builtins import str
2 import json
3 import sys
4 if sys.version_info[0] < 3:
5     from codecs import open
6 
7 script_templates = {
8     "default": """#retriever
9 from retriever.lib.templates import BasicTextTemplate
10 from retriever.lib.models import Table, Cleanup, correct_invalid_value
11 
12 SCRIPT = BasicTextTemplate(%s)""",
13 
14     "html_table": """#retriever
15 from retriever.lib.templates import HtmlTableTemplate
16 from retriever.lib.models import Table, Cleanup, correct_invalid_value
17 
18 SCRIPT = HtmlTableTemplate(%s)""",
19 }
20 
21 
22 def compile_script(script_file):
23     definition = open(script_file + ".script", 'r')
24 
25     values = {}
26     urls = {}
27     tables = {}
28     last_table = ""
29     replace = []
30     keys_to_ignore = ["template"]
31 
32     for line in [line.strip() for line in definition]:
33         if line and ':' in line and not line[0] == '#':
34             split_line = [a.strip() for a in line.split(":")]
35             key = split_line[0].lower()
36             value = ':'.join(split_line[1:])
37             if key == "table":
38                 table_name = value.split(',')[0].strip()
39                 last_table = table_name
40                 table_url = ','.join(value.split(',')[1:]).strip()
41                 urls[table_name] = table_url
42                 if replace:
43                     try:
44                         tables[last_table]
45                     except:
46                         tables[table_name] = {'replace_columns': str(replace)}
47             elif key == "*nulls":
48                 if last_table:
49                     nulls = [eval(v) for v in [v.strip()
50                                                for v in value.split(',')]]
51                     try:
52                         tables[last_table]
53                     except KeyError:
54                         if replace:
55                             tables[last_table] = {'replace_columns': str(replace)}
56                         else:
57                             tables[last_table] = {}
58                     tables[last_table]['cleanup'] = "Cleanup(correct_invalid_value, nulls=" + str(nulls) + ")"
59             elif key == "replace":
60                 replace = [(v.split(',')[0].strip(), v.split(',')[1].strip())
61                            for v in [v.strip() for v in value.split(';')]]
62             elif key == "tags":
63                 values["tags"] = [v.strip() for v in value.split(',')]
64             elif key == "*ct_names":
65                 tables[last_table]["ct_names"] = [v.strip()
66                                                   for v in value.split(',')]
67             elif key == "*column":
68                 if last_table:
69                     vs = [v.strip() for v in value.split(',')]
70                     column = [(vs[0], (vs[1], vs[2]) if len(vs) > 2 else (vs[1],))]
71                     try:
72                         tables[last_table]
73                     except KeyError:
74                         tables[last_table] = {}
75 
76                     try:
77                         tables[last_table]['columns'] += column
78                     except KeyError:
79                         tables[last_table]['columns'] = column
80             elif key[0] == "*":
81                 # attribute that should be applied to the most recently
82                 # declared table
83                 if key[0] == "*":
84                     key = key[1:]
85                 if last_table:
86                     try:
87                         tables[last_table]
88                     except KeyError:
89                         tables[last_table] = {}
90 
91                     try:
92                         e = eval(value)
93                     except:
94                         e = str(value)
95 
96                     tables[last_table][key] = "'" + str(e) + "'"
97             else:
98                 # general script attributes
99                 values[key] = '"' + str(value) + '"'
100 
101     if 'shortname' not in list(values.keys()):
102         try:
103             values['shortname'] = values['name']
104         except:
105             pass
106     values['urls'] = str(urls)
107 
108     def get_value(key):
109         try:
110             return values[key]
111         except KeyError:
112             return ""
113 
114     table_desc = "{"
115     for (key, value) in list(tables.items()):
116         table_desc += "'" + key + "': Table('" + key + "', "
117         table_desc += ','.join([key + "=" + str(value)
118                                 for key, value, in list(value.items())])
119         table_desc += "),"
120     if table_desc != '{':
121         table_desc = table_desc[:-1]
122     table_desc += "}"
123 
124     values['tables'] = table_desc
125 
126     script_desc = []
127     for key, value in list(values.items()):
128         if key == "url":
129             key = "ref"
130         if key not in keys_to_ignore:
131             script_desc.append(key + "=" + str(value))
132     script_desc = (',\n' + ' ' * 27).join(script_desc)
133 
134     if 'template' in list(values.keys()):
135         template = values["template"]
136     else:
137         template = "default"
138     script_contents = (script_templates[template] % script_desc)
139 
140     new_script = open(script_file + '.py', 'w')
141     new_script.write(script_contents)
142     new_script.close()
143 
144     definition.close()
145 
146 
147 def add_dialect(table_dict, table):
148     """
149     Reads dialect key of JSON script and extracts key-value pairs to store them
150     in python script
151 
152     Contains properties such 'nulls', delimiter', etc
153     """
154     for (key, val) in table['dialect'].items():
155         # dialect related key-value pairs
156         # copied as is
157         if key == "nulls":
158             table_dict[
159                 'cleanup'] = "Cleanup(correct_invalid_value, nulls=" + str(val) + ")"
160 
161         elif key == "delimiter":
162             table_dict[key] = "'" + str(val) + "'"
163         else:
164             table_dict[key] = val
165 
166 
167 def add_schema(table_dict, table):
168     """
169     Reads schema key of JSON script and extracts values to store them in
170     python script
171 
172     Contains properties related to table schema, such as 'fields' and cross-tab
173     column name ('ct_column').
174     """
175     for (key, val) in table['schema'].items():
176         # schema related key-value pairs
177 
178         if key == "fields":
179             # fields = columns of the table
180 
181             # list of column tuples
182             column_list = []
183             for obj in val:
184                 # fields is a collection of JSON objects
185                 # (similar to a list of dicts in python)
186 
187                 if "size" in obj:
188                     column_list.append((obj["name"],
189                                         (obj["type"], obj["size"])))
190                 else:
191                     column_list.append((obj["name"],
192                                         (obj["type"],)))
193 
194             table_dict["columns"] = column_list
195 
196         elif key == "ct_column":
197             table_dict[key] = "'" + val + "'"
198 
199         else:
200             table_dict[key] = val
201 
202 
203 def compile_json(json_file):
204     """
205     Function to compile JSON script files to python scripts
206     The scripts are created with `retriever create_json <script_name` using
207     command line
208     """
209     json_object = json.load(open(json_file + ".json", "r"))
210 
211     if "retriever" not in json_object.keys():
212         # Compile only files that have retriever key
213         return
214 
215     values = {}
216     values['urls'] = {}
217 
218     keys_to_ignore = ["template"]
219 
220     for (key, value) in json_object.items():
221 
222         if key == "title":
223             values["name"] = "\"" + str(value) + "\""
224 
225         elif key == "name":
226             values["shortname"] = "\"" + str(value) + "\""
227 
228         elif key == "description":
229             values["description"] = "\"" + str(value) + "\""
230 
231         elif key == "homepage":
232             values["ref"] = "\"" + str(value) + "\""
233 
234         elif key == "citation":
235             values["citation"] = "\"" + str(value) + "\""
236 
237         elif key == "keywords":
238             values["tags"] = value
239 
240         elif key == "retriever_minimum_version":
241             values["retriever_minimum_version"] = "\"" + value + "\""
242 
243         elif key == "resources":
244             # Array of table objects
245             tables = {}
246             for table in value:
247                 # Maintain a dict for table keys and values
248                 table_dict = {}
249 
250                 try:
251                     values['urls'][table['name']] = table['url']
252                 except Exception as e:
253                     print(e, "\nError in reading table: " + table)
254                     continue
255 
256                 if table["schema"] == {} and table["dialect"] == {}:
257                     continue
258 
259                 for (t_key, t_val) in table.items():
260 
261                     if t_key == "dialect":
262                         add_dialect(table_dict, table)
263 
264                     elif t_key == "schema":
265                         add_schema(table_dict, table)
266 
267                 tables[table["name"]] = table_dict
268 
269         else:
270             values[key] = value
271 
272     # Create a Table object string using the tables dict
273     table_desc = "{"
274     for (key, value) in tables.items():
275         table_desc += "'" + key + "': Table('" + key + "', "
276         table_desc += ','.join([key + "=" + str(value)
277                                 for key, value, in value.items()])
278         table_desc += "),"
279     if table_desc != '{':
280         table_desc = table_desc[:-1]
281     table_desc += "}"
282 
283     values["tables"] = table_desc
284 
285     script_desc = []
286     for key, value in values.items():
287         if key not in keys_to_ignore:
288             script_desc.append(key + "=" + str(value))
289     script_desc = (',\n' + ' ' * 27).join(script_desc)
290 
291     if 'template' in values.keys():
292         template = values["template"]
293     else:
294         template = "default"
295     script_contents = (script_templates[template] % script_desc)
296 
297     new_script = open(json_file + '.py', 'w', encoding='utf-8')
298     new_script.write('# -*- coding: latin-1 -*-\n')
299     new_script.write(script_contents)
300     new_script.close()
301
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/lib/compile.py b/lib/compile.py
--- a/lib/compile.py
+++ b/lib/compile.py
@@ -238,7 +238,7 @@
             values["tags"] = value
 
         elif key == "retriever_minimum_version":
-            values["retriever_minimum_version"] = "\"" + value + "\""
+            values["retriever_minimum_version"] = "\"" + str(value) + "\""
 
         elif key == "resources":
             # Array of table objects

{"golden_diff": "diff --git a/lib/compile.py b/lib/compile.py\n--- a/lib/compile.py\n+++ b/lib/compile.py\n@@ -238,7 +238,7 @@\n             values[\"tags\"] = value\n \n         elif key == \"retriever_minimum_version\":\n-            values[\"retriever_minimum_version\"] = \"\\\"\" + value + \"\\\"\"\n+            values[\"retriever_minimum_version\"] = \"\\\"\" + str(value) + \"\\\"\"\n \n         elif key == \"resources\":\n             # Array of table objects\n", "issue": "Stop bad scripts from causing errors\nCurrently when `compile_json` gets run if something goes wrong the retriever errors out even on commands not running the script (e.g., `retriever ls`). What it should do is ignore the bad script and possibly report back that there is an issue with the script but keep running normally otherwise.\n\n", "before_files": [{"content": "from builtins import str\nimport json\nimport sys\nif sys.version_info[0] < 3:\n    from codecs import open\n\nscript_templates = {\n    \"default\": \"\"\"#retriever\nfrom retriever.lib.templates import BasicTextTemplate\nfrom retriever.lib.models import Table, Cleanup, correct_invalid_value\n\nSCRIPT = BasicTextTemplate(%s)\"\"\",\n\n    \"html_table\": \"\"\"#retriever\nfrom retriever.lib.templates import HtmlTableTemplate\nfrom retriever.lib.models import Table, Cleanup, correct_invalid_value\n\nSCRIPT = HtmlTableTemplate(%s)\"\"\",\n}\n\n\ndef compile_script(script_file):\n    definition = open(script_file + \".script\", 'r')\n\n    values = {}\n    urls = {}\n    tables = {}\n    last_table = \"\"\n    replace = []\n    keys_to_ignore = [\"template\"]\n\n    for line in [line.strip() for line in definition]:\n        if line and ':' in line and not line[0] == '#':\n            split_line = [a.strip() for a in line.split(\":\")]\n            key = split_line[0].lower()\n            value = ':'.join(split_line[1:])\n            if key == \"table\":\n                table_name = value.split(',')[0].strip()\n                last_table = table_name\n                table_url = ','.join(value.split(',')[1:]).strip()\n                urls[table_name] = table_url\n                if replace:\n                    try:\n                        tables[last_table]\n                    except:\n                        tables[table_name] = {'replace_columns': str(replace)}\n            elif key == \"*nulls\":\n                if last_table:\n                    nulls = [eval(v) for v in [v.strip()\n                                               for v in value.split(',')]]\n                    try:\n                        tables[last_table]\n                    except KeyError:\n                        if replace:\n                            tables[last_table] = {'replace_columns': str(replace)}\n                        else:\n                            tables[last_table] = {}\n                    tables[last_table]['cleanup'] = \"Cleanup(correct_invalid_value, nulls=\" + str(nulls) + \")\"\n            elif key == \"replace\":\n                replace = [(v.split(',')[0].strip(), v.split(',')[1].strip())\n                           for v in [v.strip() for v in value.split(';')]]\n            elif key == \"tags\":\n                values[\"tags\"] = [v.strip() for v in value.split(',')]\n            elif key == \"*ct_names\":\n                tables[last_table][\"ct_names\"] = [v.strip()\n                                                  for v in value.split(',')]\n            elif key == \"*column\":\n                if last_table:\n                    vs = [v.strip() for v in value.split(',')]\n                    column = [(vs[0], (vs[1], vs[2]) if len(vs) > 2 else (vs[1],))]\n                    try:\n                        tables[last_table]\n                    except KeyError:\n                        tables[last_table] = {}\n\n                    try:\n                        tables[last_table]['columns'] += column\n                    except KeyError:\n                        tables[last_table]['columns'] = column\n            elif key[0] == \"*\":\n                # attribute that should be applied to the most recently\n                # declared table\n                if key[0] == \"*\":\n                    key = key[1:]\n                if last_table:\n                    try:\n                        tables[last_table]\n                    except KeyError:\n                        tables[last_table] = {}\n\n                    try:\n                        e = eval(value)\n                    except:\n                        e = str(value)\n\n                    tables[last_table][key] = \"'\" + str(e) + \"'\"\n            else:\n                # general script attributes\n                values[key] = '\"' + str(value) + '\"'\n\n    if 'shortname' not in list(values.keys()):\n        try:\n            values['shortname'] = values['name']\n        except:\n            pass\n    values['urls'] = str(urls)\n\n    def get_value(key):\n        try:\n            return values[key]\n        except KeyError:\n            return \"\"\n\n    table_desc = \"{\"\n    for (key, value) in list(tables.items()):\n        table_desc += \"'\" + key + \"': Table('\" + key + \"', \"\n        table_desc += ','.join([key + \"=\" + str(value)\n                                for key, value, in list(value.items())])\n        table_desc += \"),\"\n    if table_desc != '{':\n        table_desc = table_desc[:-1]\n    table_desc += \"}\"\n\n    values['tables'] = table_desc\n\n    script_desc = []\n    for key, value in list(values.items()):\n        if key == \"url\":\n            key = \"ref\"\n        if key not in keys_to_ignore:\n            script_desc.append(key + \"=\" + str(value))\n    script_desc = (',\\n' + ' ' * 27).join(script_desc)\n\n    if 'template' in list(values.keys()):\n        template = values[\"template\"]\n    else:\n        template = \"default\"\n    script_contents = (script_templates[template] % script_desc)\n\n    new_script = open(script_file + '.py', 'w')\n    new_script.write(script_contents)\n    new_script.close()\n\n    definition.close()\n\n\ndef add_dialect(table_dict, table):\n    \"\"\"\n    Reads dialect key of JSON script and extracts key-value pairs to store them\n    in python script\n\n    Contains properties such 'nulls', delimiter', etc\n    \"\"\"\n    for (key, val) in table['dialect'].items():\n        # dialect related key-value pairs\n        # copied as is\n        if key == \"nulls\":\n            table_dict[\n                'cleanup'] = \"Cleanup(correct_invalid_value, nulls=\" + str(val) + \")\"\n\n        elif key == \"delimiter\":\n            table_dict[key] = \"'\" + str(val) + \"'\"\n        else:\n            table_dict[key] = val\n\n\ndef add_schema(table_dict, table):\n    \"\"\"\n    Reads schema key of JSON script and extracts values to store them in\n    python script\n\n    Contains properties related to table schema, such as 'fields' and cross-tab\n    column name ('ct_column').\n    \"\"\"\n    for (key, val) in table['schema'].items():\n        # schema related key-value pairs\n\n        if key == \"fields\":\n            # fields = columns of the table\n\n            # list of column tuples\n            column_list = []\n            for obj in val:\n                # fields is a collection of JSON objects\n                # (similar to a list of dicts in python)\n\n                if \"size\" in obj:\n                    column_list.append((obj[\"name\"],\n                                        (obj[\"type\"], obj[\"size\"])))\n                else:\n                    column_list.append((obj[\"name\"],\n                                        (obj[\"type\"],)))\n\n            table_dict[\"columns\"] = column_list\n\n        elif key == \"ct_column\":\n            table_dict[key] = \"'\" + val + \"'\"\n\n        else:\n            table_dict[key] = val\n\n\ndef compile_json(json_file):\n    \"\"\"\n    Function to compile JSON script files to python scripts\n    The scripts are created with `retriever create_json <script_name` using\n    command line\n    \"\"\"\n    json_object = json.load(open(json_file + \".json\", \"r\"))\n\n    if \"retriever\" not in json_object.keys():\n        # Compile only files that have retriever key\n        return\n\n    values = {}\n    values['urls'] = {}\n\n    keys_to_ignore = [\"template\"]\n\n    for (key, value) in json_object.items():\n\n        if key == \"title\":\n            values[\"name\"] = \"\\\"\" + str(value) + \"\\\"\"\n\n        elif key == \"name\":\n            values[\"shortname\"] = \"\\\"\" + str(value) + \"\\\"\"\n\n        elif key == \"description\":\n            values[\"description\"] = \"\\\"\" + str(value) + \"\\\"\"\n\n        elif key == \"homepage\":\n            values[\"ref\"] = \"\\\"\" + str(value) + \"\\\"\"\n\n        elif key == \"citation\":\n            values[\"citation\"] = \"\\\"\" + str(value) + \"\\\"\"\n\n        elif key == \"keywords\":\n            values[\"tags\"] = value\n\n        elif key == \"retriever_minimum_version\":\n            values[\"retriever_minimum_version\"] = \"\\\"\" + value + \"\\\"\"\n\n        elif key == \"resources\":\n            # Array of table objects\n            tables = {}\n            for table in value:\n                # Maintain a dict for table keys and values\n                table_dict = {}\n\n                try:\n                    values['urls'][table['name']] = table['url']\n                except Exception as e:\n                    print(e, \"\\nError in reading table: \" + table)\n                    continue\n\n                if table[\"schema\"] == {} and table[\"dialect\"] == {}:\n                    continue\n\n                for (t_key, t_val) in table.items():\n\n                    if t_key == \"dialect\":\n                        add_dialect(table_dict, table)\n\n                    elif t_key == \"schema\":\n                        add_schema(table_dict, table)\n\n                tables[table[\"name\"]] = table_dict\n\n        else:\n            values[key] = value\n\n    # Create a Table object string using the tables dict\n    table_desc = \"{\"\n    for (key, value) in tables.items():\n        table_desc += \"'\" + key + \"': Table('\" + key + \"', \"\n        table_desc += ','.join([key + \"=\" + str(value)\n                                for key, value, in value.items()])\n        table_desc += \"),\"\n    if table_desc != '{':\n        table_desc = table_desc[:-1]\n    table_desc += \"}\"\n\n    values[\"tables\"] = table_desc\n\n    script_desc = []\n    for key, value in values.items():\n        if key not in keys_to_ignore:\n            script_desc.append(key + \"=\" + str(value))\n    script_desc = (',\\n' + ' ' * 27).join(script_desc)\n\n    if 'template' in values.keys():\n        template = values[\"template\"]\n    else:\n        template = \"default\"\n    script_contents = (script_templates[template] % script_desc)\n\n    new_script = open(json_file + '.py', 'w', encoding='utf-8')\n    new_script.write('# -*- coding: latin-1 -*-\\n')\n    new_script.write(script_contents)\n    new_script.close()\n", "path": "lib/compile.py"}], "after_files": [{"content": "from builtins import str\nimport json\nimport sys\nif sys.version_info[0] < 3:\n    from codecs import open\n\nscript_templates = {\n    \"default\": \"\"\"#retriever\nfrom retriever.lib.templates import BasicTextTemplate\nfrom retriever.lib.models import Table, Cleanup, correct_invalid_value\n\nSCRIPT = BasicTextTemplate(%s)\"\"\",\n\n    \"html_table\": \"\"\"#retriever\nfrom retriever.lib.templates import HtmlTableTemplate\nfrom retriever.lib.models import Table, Cleanup, correct_invalid_value\n\nSCRIPT = HtmlTableTemplate(%s)\"\"\",\n}\n\n\ndef compile_script(script_file):\n    definition = open(script_file + \".script\", 'r')\n\n    values = {}\n    urls = {}\n    tables = {}\n    last_table = \"\"\n    replace = []\n    keys_to_ignore = [\"template\"]\n\n    for line in [line.strip() for line in definition]:\n        if line and ':' in line and not line[0] == '#':\n            split_line = [a.strip() for a in line.split(\":\")]\n            key = split_line[0].lower()\n            value = ':'.join(split_line[1:])\n            if key == \"table\":\n                table_name = value.split(',')[0].strip()\n                last_table = table_name\n                table_url = ','.join(value.split(',')[1:]).strip()\n                urls[table_name] = table_url\n                if replace:\n                    try:\n                        tables[last_table]\n                    except:\n                        tables[table_name] = {'replace_columns': str(replace)}\n            elif key == \"*nulls\":\n                if last_table:\n                    nulls = [eval(v) for v in [v.strip()\n                                               for v in value.split(',')]]\n                    try:\n                        tables[last_table]\n                    except KeyError:\n                        if replace:\n                            tables[last_table] = {'replace_columns': str(replace)}\n                        else:\n                            tables[last_table] = {}\n                    tables[last_table]['cleanup'] = \"Cleanup(correct_invalid_value, nulls=\" + str(nulls) + \")\"\n            elif key == \"replace\":\n                replace = [(v.split(',')[0].strip(), v.split(',')[1].strip())\n                           for v in [v.strip() for v in value.split(';')]]\n            elif key == \"tags\":\n                values[\"tags\"] = [v.strip() for v in value.split(',')]\n            elif key == \"*ct_names\":\n                tables[last_table][\"ct_names\"] = [v.strip()\n                                                  for v in value.split(',')]\n            elif key == \"*column\":\n                if last_table:\n                    vs = [v.strip() for v in value.split(',')]\n                    column = [(vs[0], (vs[1], vs[2]) if len(vs) > 2 else (vs[1],))]\n                    try:\n                        tables[last_table]\n                    except KeyError:\n                        tables[last_table] = {}\n\n                    try:\n                        tables[last_table]['columns'] += column\n                    except KeyError:\n                        tables[last_table]['columns'] = column\n            elif key[0] == \"*\":\n                # attribute that should be applied to the most recently\n                # declared table\n                if key[0] == \"*\":\n                    key = key[1:]\n                if last_table:\n                    try:\n                        tables[last_table]\n                    except KeyError:\n                        tables[last_table] = {}\n\n                    try:\n                        e = eval(value)\n                    except:\n                        e = str(value)\n\n                    tables[last_table][key] = \"'\" + str(e) + \"'\"\n            else:\n                # general script attributes\n                values[key] = '\"' + str(value) + '\"'\n\n    if 'shortname' not in list(values.keys()):\n        try:\n            values['shortname'] = values['name']\n        except:\n            pass\n    values['urls'] = str(urls)\n\n    def get_value(key):\n        try:\n            return values[key]\n        except KeyError:\n            return \"\"\n\n    table_desc = \"{\"\n    for (key, value) in list(tables.items()):\n        table_desc += \"'\" + key + \"': Table('\" + key + \"', \"\n        table_desc += ','.join([key + \"=\" + str(value)\n                                for key, value, in list(value.items())])\n        table_desc += \"),\"\n    if table_desc != '{':\n        table_desc = table_desc[:-1]\n    table_desc += \"}\"\n\n    values['tables'] = table_desc\n\n    script_desc = []\n    for key, value in list(values.items()):\n        if key == \"url\":\n            key = \"ref\"\n        if key not in keys_to_ignore:\n            script_desc.append(key + \"=\" + str(value))\n    script_desc = (',\\n' + ' ' * 27).join(script_desc)\n\n    if 'template' in list(values.keys()):\n        template = values[\"template\"]\n    else:\n        template = \"default\"\n    script_contents = (script_templates[template] % script_desc)\n\n    new_script = open(script_file + '.py', 'w')\n    new_script.write(script_contents)\n    new_script.close()\n\n    definition.close()\n\n\ndef add_dialect(table_dict, table):\n    \"\"\"\n    Reads dialect key of JSON script and extracts key-value pairs to store them\n    in python script\n\n    Contains properties such 'nulls', delimiter', etc\n    \"\"\"\n    for (key, val) in table['dialect'].items():\n        # dialect related key-value pairs\n        # copied as is\n        if key == \"nulls\":\n            table_dict[\n                'cleanup'] = \"Cleanup(correct_invalid_value, nulls=\" + str(val) + \")\"\n\n        elif key == \"delimiter\":\n            table_dict[key] = \"'\" + str(val) + \"'\"\n        else:\n            table_dict[key] = val\n\n\ndef add_schema(table_dict, table):\n    \"\"\"\n    Reads schema key of JSON script and extracts values to store them in\n    python script\n\n    Contains properties related to table schema, such as 'fields' and cross-tab\n    column name ('ct_column').\n    \"\"\"\n    for (key, val) in table['schema'].items():\n        # schema related key-value pairs\n\n        if key == \"fields\":\n            # fields = columns of the table\n\n            # list of column tuples\n            column_list = []\n            for obj in val:\n                # fields is a collection of JSON objects\n                # (similar to a list of dicts in python)\n\n                if \"size\" in obj:\n                    column_list.append((obj[\"name\"],\n                                        (obj[\"type\"], obj[\"size\"])))\n                else:\n                    column_list.append((obj[\"name\"],\n                                        (obj[\"type\"],)))\n\n            table_dict[\"columns\"] = column_list\n\n        elif key == \"ct_column\":\n            table_dict[key] = \"'\" + val + \"'\"\n\n        else:\n            table_dict[key] = val\n\n\ndef compile_json(json_file):\n    \"\"\"\n    Function to compile JSON script files to python scripts\n    The scripts are created with `retriever create_json <script_name` using\n    command line\n    \"\"\"\n    json_object = json.load(open(json_file + \".json\", \"r\"))\n\n    if \"retriever\" not in json_object.keys():\n        # Compile only files that have retriever key\n        return\n\n    values = {}\n    values['urls'] = {}\n\n    keys_to_ignore = [\"template\"]\n\n    for (key, value) in json_object.items():\n\n        if key == \"title\":\n            values[\"name\"] = \"\\\"\" + str(value) + \"\\\"\"\n\n        elif key == \"name\":\n            values[\"shortname\"] = \"\\\"\" + str(value) + \"\\\"\"\n\n        elif key == \"description\":\n            values[\"description\"] = \"\\\"\" + str(value) + \"\\\"\"\n\n        elif key == \"homepage\":\n            values[\"ref\"] = \"\\\"\" + str(value) + \"\\\"\"\n\n        elif key == \"citation\":\n            values[\"citation\"] = \"\\\"\" + str(value) + \"\\\"\"\n\n        elif key == \"keywords\":\n            values[\"tags\"] = value\n\n        elif key == \"retriever_minimum_version\":\n            values[\"retriever_minimum_version\"] = \"\\\"\" + str(value) + \"\\\"\"\n\n        elif key == \"resources\":\n            # Array of table objects\n            tables = {}\n            for table in value:\n                # Maintain a dict for table keys and values\n                table_dict = {}\n\n                try:\n                    values['urls'][table['name']] = table['url']\n                except Exception as e:\n                    print(e, \"\\nError in reading table: \" + table)\n                    continue\n\n                if table[\"schema\"] == {} and table[\"dialect\"] == {}:\n                    continue\n\n                for (t_key, t_val) in table.items():\n\n                    if t_key == \"dialect\":\n                        add_dialect(table_dict, table)\n\n                    elif t_key == \"schema\":\n                        add_schema(table_dict, table)\n\n                tables[table[\"name\"]] = table_dict\n\n        else:\n            values[key] = value\n\n    # Create a Table object string using the tables dict\n    table_desc = \"{\"\n    for (key, value) in tables.items():\n        table_desc += \"'\" + key + \"': Table('\" + key + \"', \"\n        table_desc += ','.join([key + \"=\" + str(value)\n                                for key, value, in value.items()])\n        table_desc += \"),\"\n    if table_desc != '{':\n        table_desc = table_desc[:-1]\n    table_desc += \"}\"\n\n    values[\"tables\"] = table_desc\n\n    script_desc = []\n    for key, value in values.items():\n        if key not in keys_to_ignore:\n            script_desc.append(key + \"=\" + str(value))\n    script_desc = (',\\n' + ' ' * 27).join(script_desc)\n\n    if 'template' in values.keys():\n        template = values[\"template\"]\n    else:\n        template = \"default\"\n    script_contents = (script_templates[template] % script_desc)\n\n    new_script = open(json_file + '.py', 'w', encoding='utf-8')\n    new_script.write('# -*- coding: latin-1 -*-\\n')\n    new_script.write(script_contents)\n    new_script.close()\n", "path": "lib/compile.py"}]}