We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Gentry url update
The url given for the gentry dataset at 

https://github.com/weecology/retriever/blob/master/scripts/gentry.py#L33

Should be changed from http://www.wlbcenter.org/gentry_data.htm (which no longer seems to exist) to http://www.mobot.org/mobot/research/gentry/welcome.shtml


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scripts/gentry.py`
Content:
```
1 #retriever
2 """Retriever script for Alwyn H. Gentry Forest Transect Dataset
3 
4 """
5 from __future__ import print_function
6 from builtins import str
7 from builtins import range
8 
9 import os
10 import sys
11 import zipfile
12 import xlrd
13 from retriever.lib.templates import Script
14 from retriever.lib.models import Table
15 from retriever.lib.excel import Excel
16 
17 VERSION = '1.0.1'
18 
19 TAX_GROUPS = 9756 #9819
20 
21 
22 class main(Script):
23     def __init__(self, **kwargs):
24         Script.__init__(self, **kwargs)
25         self.name = "Alwyn H. Gentry Forest Transect Dataset"
26         self.shortname = "Gentry"
27         self.urls = {"stems": "http://www.mobot.org/mobot/gentry/123/all_Excel.zip",
28                      "sites": "http://www.ecologicaldata.org/sites/default/files/gentry_sites_data.txt",
29                      "species": "",
30                      "counts": ""}
31         self.tags = ["Taxon > Plants", "Spatial Scale > Global",
32                      "Data Type > Observational"]
33         self.ref = "http://www.wlbcenter.org/gentry_data.htm"
34         self.addendum = """Researchers who make use of the data in publications are requested to acknowledge Alwyn H. Gentry, the Missouri Botanical Garden, and collectors who assisted Gentry or contributed data for specific sites. It is also requested that a reprint of any publication making use of the Gentry Forest Transect Data be sent to:
35 
36 Bruce E. Ponman
37 Missouri Botanical Garden
38 P.O. Box 299
39 St. Louis, MO 63166-0299
40 U.S.A. """
41 
42     def download(self, engine=None, debug=False):
43         Script.download(self, engine, debug)
44 
45         self.engine.auto_create_table(Table("sites"), url=self.urls["sites"])
46         self.engine.insert_data_from_url(self.urls["sites"])
47 
48         self.engine.download_file(self.urls["stems"], "all_Excel.zip")
49         local_zip = zipfile.ZipFile(self.engine.format_filename("all_Excel.zip"))
50         filelist = local_zip.namelist()
51         local_zip.close()
52         self.engine.download_files_from_archive(self.urls["stems"], filelist)
53 
54         filelist = [os.path.basename(filename) for filename in filelist]
55 
56         # Currently all_Excel.zip is missing CURUYUQU.xls
57         # Download it separately and add it to the file list
58         if not self.engine.find_file('CURUYUQU.xls'):
59             self.engine.download_file("http://www.mobot.org/mobot/gentry/123/samerica/CURUYUQU.xls", "CURUYUQU.xls")
60             filelist.append('CURUYUQU.xls')
61 
62         lines = []
63         tax = []
64         for filename in filelist:
65             print("Extracting data from " + filename + "...")
66             book = xlrd.open_workbook(self.engine.format_filename(filename))
67             sh = book.sheet_by_index(0)
68             rows = sh.nrows
69             cn = {'stems': []}
70             n = 0
71             for colnum, c in enumerate(sh.row(0)):
72                 if not Excel.empty_cell(c):
73                     cid = Excel.cell_value(c).lower()
74                     # line number column is sometimes named differently
75                     if cid in ["sub", "number"]:
76                         cid = "line"
77                     # the "number of individuals" column is named in various
78                     # different ways; they always at least contain "nd"
79                     if "nd" in cid:
80                         cid = "count"
81                     # in QUIAPACA.xls the "number of individuals" column is
82                     # misnamed "STEMDBH" just like the stems columns, so weep
83                     # for the state of scientific data and then fix manually
84                     if filename == "QUIAPACA.xls" and colnum == 13:
85                         cid = "count"
86 
87                     # if column is a stem, add it to the list of stems;
88                     # otherwise, make note of the column name/number
89                     if "stem" in cid or "dbh" in cid:
90                         cn["stems"].append(n)
91                     else:
92                         cn[cid] = n
93                 n += 1
94             # sometimes, a data file does not contain a liana or count column
95             if not "liana" in list(cn.keys()):
96                 cn["liana"] = -1
97             if not "count" in list(cn.keys()):
98                 cn["count"] = -1
99             for i in range(1, rows):
100                 row = sh.row(i)
101                 cellcount = len(row)
102                 # make sure the row is real, not just empty cells
103                 if not all(Excel.empty_cell(cell) for cell in row):
104                     try:
105                         this_line = {}
106 
107                         def format_value(s):
108                             s = Excel.cell_value(s)
109                             return str(s).title().replace("\\", "/").replace('"', '')
110 
111                         # get the following information from the appropriate columns
112                         for i in ["line", "family", "genus", "species",
113                                   "liana", "count"]:
114                             if cn[i] > -1:
115                                 this_line[i] = format_value(row[cn[i]])
116                                 if this_line[i] == '`':
117                                     this_line[i] = 1
118 
119                         this_line["stems"] = [Excel.cell_value(row[c])
120                                               for c in cn["stems"]
121                                               if not Excel.empty_cell(row[c])]
122                         this_line["site"] = filename[0:-4]
123 
124                         lines.append(this_line)
125 
126                         # Check how far the species is identified
127                         full_id = 0
128                         if len(this_line["species"]) < 3:
129                             if len(this_line["genus"]) < 3:
130                                 id_level = "family"
131                             else:
132                                 id_level = "genus"
133                         else:
134                             id_level = "species"
135                             full_id = 1
136                         tax.append((this_line["family"],
137                                     this_line["genus"],
138                                     this_line["species"].lower().replace('\\', '').replace('"', ''),
139                                     id_level,
140                                     str(full_id)))
141                     except:
142                         raise
143                         pass
144 
145         tax = sorted(tax, key=lambda group: group[0] + " " + group[1] + " " + group[2])
146         unique_tax = []
147         tax_dict = dict()
148         tax_count = 0
149 
150         # Get all unique families/genera/species
151         for group in tax:
152             if not (group in unique_tax):
153                 unique_tax.append(group)
154                 tax_count += 1
155                 tax_dict[group[0:3]] = tax_count
156                 if tax_count % 10 == 0:
157                     msg = "Generating taxonomic groups: " + str(tax_count) + " / " + str(TAX_GROUPS)
158                     sys.stdout.write(msg + "\b" * len(msg))
159         print("Generating taxonomic groups: " + str(TAX_GROUPS) + " / " + str(TAX_GROUPS))
160 
161 
162         # Create species table
163         table = Table("species", delimiter=",")
164         table.columns=[("species_id"            ,   ("pk-int",)    ),
165                        ("family"                ,   ("char", )    ),
166                        ("genus"                 ,   ("char", )    ),
167                        ("species"               ,   ("char", )    ),
168                        ("id_level"              ,   ("char", 10)    ),
169                        ("full_id"               ,   ("bool",)       )]
170 
171         data = [','.join([str(tax_dict[group[:3]])] + ['"%s"' % g for g in group])
172                 for group in unique_tax]
173         table.pk = 'species_id'
174         table.contains_pk = True
175 
176         self.engine.table = table
177         self.engine.create_table()
178         self.engine.add_to_table(data)
179 
180 
181         # Create stems table
182         table = Table("stems", delimiter=",", contains_pk=False)
183         table.columns=[("stem_id"               ,   ("pk-auto",)    ),
184                        ("line"                  ,   ("int",)        ),
185                        ("species_id"            ,   ("int",)        ),
186                        ("site_code"             ,   ("char", 12)    ),
187                        ("liana"                 ,   ("char", 10)    ),
188                        ("stem"                  ,   ("double",)     )]
189         stems = []
190         counts = []
191         for line in lines:
192             try:
193                 liana = line["liana"]
194             except KeyError:
195                 liana = ""
196             species_info = [line["line"],
197                             tax_dict[(line["family"],
198                                       line["genus"],
199                                       line["species"].lower())],
200                             line["site"],
201                             liana
202                             ]
203             try:
204                 counts.append([str(value) for value in species_info + [line["count"]]])
205             except KeyError:
206                 pass
207 
208             for i in line["stems"]:
209                 stem = species_info + [i]
210                 stems.append([str(value) for value in stem])
211 
212         data = [','.join(stem) for stem in stems]
213         self.engine.table = table
214         self.engine.create_table()
215         self.engine.add_to_table(data)
216 
217 
218         # Create counts table
219         table = Table("counts", delimiter=",", contains_pk=False)
220         table.columns=[("count_id"              ,   ("pk-auto",)    ),
221                        ("line"                  ,   ("int",)        ),
222                        ("species_id"            ,   ("int",)        ),
223                        ("site_code"             ,   ("char", 12)    ),
224                        ("liana"                 ,   ("char", 10)    ),
225                        ("count"                 ,   ("double",)     )]
226         data = [','.join(count) for count in counts]
227         self.engine.table = table
228         self.engine.create_table()
229         self.engine.add_to_table(data)
230 
231         return self.engine
232 
233 
234 SCRIPT = main()
235
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scripts/gentry.py b/scripts/gentry.py
--- a/scripts/gentry.py
+++ b/scripts/gentry.py
@@ -30,7 +30,7 @@
                      "counts": ""}
         self.tags = ["Taxon > Plants", "Spatial Scale > Global",
                      "Data Type > Observational"]
-        self.ref = "http://www.wlbcenter.org/gentry_data.htm"
+        self.ref = "http://www.mobot.org/mobot/research/gentry/welcome.shtml"
         self.addendum = """Researchers who make use of the data in publications are requested to acknowledge Alwyn H. Gentry, the Missouri Botanical Garden, and collectors who assisted Gentry or contributed data for specific sites. It is also requested that a reprint of any publication making use of the Gentry Forest Transect Data be sent to:
 
 Bruce E. Ponman

{"golden_diff": "diff --git a/scripts/gentry.py b/scripts/gentry.py\n--- a/scripts/gentry.py\n+++ b/scripts/gentry.py\n@@ -30,7 +30,7 @@\n                      \"counts\": \"\"}\n         self.tags = [\"Taxon > Plants\", \"Spatial Scale > Global\",\n                      \"Data Type > Observational\"]\n-        self.ref = \"http://www.wlbcenter.org/gentry_data.htm\"\n+        self.ref = \"http://www.mobot.org/mobot/research/gentry/welcome.shtml\"\n         self.addendum = \"\"\"Researchers who make use of the data in publications are requested to acknowledge Alwyn H. Gentry, the Missouri Botanical Garden, and collectors who assisted Gentry or contributed data for specific sites. It is also requested that a reprint of any publication making use of the Gentry Forest Transect Data be sent to:\n \n Bruce E. Ponman\n", "issue": "Gentry url update\nThe url given for the gentry dataset at \n\nhttps://github.com/weecology/retriever/blob/master/scripts/gentry.py#L33\n\nShould be changed from http://www.wlbcenter.org/gentry_data.htm (which no longer seems to exist) to http://www.mobot.org/mobot/research/gentry/welcome.shtml\n\n", "before_files": [{"content": "#retriever\n\"\"\"Retriever script for Alwyn H. Gentry Forest Transect Dataset\n\n\"\"\"\nfrom __future__ import print_function\nfrom builtins import str\nfrom builtins import range\n\nimport os\nimport sys\nimport zipfile\nimport xlrd\nfrom retriever.lib.templates import Script\nfrom retriever.lib.models import Table\nfrom retriever.lib.excel import Excel\n\nVERSION = '1.0.1'\n\nTAX_GROUPS = 9756 #9819\n\n\nclass main(Script):\n    def __init__(self, **kwargs):\n        Script.__init__(self, **kwargs)\n        self.name = \"Alwyn H. Gentry Forest Transect Dataset\"\n        self.shortname = \"Gentry\"\n        self.urls = {\"stems\": \"http://www.mobot.org/mobot/gentry/123/all_Excel.zip\",\n                     \"sites\": \"http://www.ecologicaldata.org/sites/default/files/gentry_sites_data.txt\",\n                     \"species\": \"\",\n                     \"counts\": \"\"}\n        self.tags = [\"Taxon > Plants\", \"Spatial Scale > Global\",\n                     \"Data Type > Observational\"]\n        self.ref = \"http://www.wlbcenter.org/gentry_data.htm\"\n        self.addendum = \"\"\"Researchers who make use of the data in publications are requested to acknowledge Alwyn H. Gentry, the Missouri Botanical Garden, and collectors who assisted Gentry or contributed data for specific sites. It is also requested that a reprint of any publication making use of the Gentry Forest Transect Data be sent to:\n\nBruce E. Ponman\nMissouri Botanical Garden\nP.O. Box 299\nSt. Louis, MO 63166-0299\nU.S.A. \"\"\"\n\n    def download(self, engine=None, debug=False):\n        Script.download(self, engine, debug)\n\n        self.engine.auto_create_table(Table(\"sites\"), url=self.urls[\"sites\"])\n        self.engine.insert_data_from_url(self.urls[\"sites\"])\n\n        self.engine.download_file(self.urls[\"stems\"], \"all_Excel.zip\")\n        local_zip = zipfile.ZipFile(self.engine.format_filename(\"all_Excel.zip\"))\n        filelist = local_zip.namelist()\n        local_zip.close()\n        self.engine.download_files_from_archive(self.urls[\"stems\"], filelist)\n\n        filelist = [os.path.basename(filename) for filename in filelist]\n\n        # Currently all_Excel.zip is missing CURUYUQU.xls\n        # Download it separately and add it to the file list\n        if not self.engine.find_file('CURUYUQU.xls'):\n            self.engine.download_file(\"http://www.mobot.org/mobot/gentry/123/samerica/CURUYUQU.xls\", \"CURUYUQU.xls\")\n            filelist.append('CURUYUQU.xls')\n\n        lines = []\n        tax = []\n        for filename in filelist:\n            print(\"Extracting data from \" + filename + \"...\")\n            book = xlrd.open_workbook(self.engine.format_filename(filename))\n            sh = book.sheet_by_index(0)\n            rows = sh.nrows\n            cn = {'stems': []}\n            n = 0\n            for colnum, c in enumerate(sh.row(0)):\n                if not Excel.empty_cell(c):\n                    cid = Excel.cell_value(c).lower()\n                    # line number column is sometimes named differently\n                    if cid in [\"sub\", \"number\"]:\n                        cid = \"line\"\n                    # the \"number of individuals\" column is named in various\n                    # different ways; they always at least contain \"nd\"\n                    if \"nd\" in cid:\n                        cid = \"count\"\n                    # in QUIAPACA.xls the \"number of individuals\" column is\n                    # misnamed \"STEMDBH\" just like the stems columns, so weep\n                    # for the state of scientific data and then fix manually\n                    if filename == \"QUIAPACA.xls\" and colnum == 13:\n                        cid = \"count\"\n\n                    # if column is a stem, add it to the list of stems;\n                    # otherwise, make note of the column name/number\n                    if \"stem\" in cid or \"dbh\" in cid:\n                        cn[\"stems\"].append(n)\n                    else:\n                        cn[cid] = n\n                n += 1\n            # sometimes, a data file does not contain a liana or count column\n            if not \"liana\" in list(cn.keys()):\n                cn[\"liana\"] = -1\n            if not \"count\" in list(cn.keys()):\n                cn[\"count\"] = -1\n            for i in range(1, rows):\n                row = sh.row(i)\n                cellcount = len(row)\n                # make sure the row is real, not just empty cells\n                if not all(Excel.empty_cell(cell) for cell in row):\n                    try:\n                        this_line = {}\n\n                        def format_value(s):\n                            s = Excel.cell_value(s)\n                            return str(s).title().replace(\"\\\\\", \"/\").replace('\"', '')\n\n                        # get the following information from the appropriate columns\n                        for i in [\"line\", \"family\", \"genus\", \"species\",\n                                  \"liana\", \"count\"]:\n                            if cn[i] > -1:\n                                this_line[i] = format_value(row[cn[i]])\n                                if this_line[i] == '`':\n                                    this_line[i] = 1\n\n                        this_line[\"stems\"] = [Excel.cell_value(row[c])\n                                              for c in cn[\"stems\"]\n                                              if not Excel.empty_cell(row[c])]\n                        this_line[\"site\"] = filename[0:-4]\n\n                        lines.append(this_line)\n\n                        # Check how far the species is identified\n                        full_id = 0\n                        if len(this_line[\"species\"]) < 3:\n                            if len(this_line[\"genus\"]) < 3:\n                                id_level = \"family\"\n                            else:\n                                id_level = \"genus\"\n                        else:\n                            id_level = \"species\"\n                            full_id = 1\n                        tax.append((this_line[\"family\"],\n                                    this_line[\"genus\"],\n                                    this_line[\"species\"].lower().replace('\\\\', '').replace('\"', ''),\n                                    id_level,\n                                    str(full_id)))\n                    except:\n                        raise\n                        pass\n\n        tax = sorted(tax, key=lambda group: group[0] + \" \" + group[1] + \" \" + group[2])\n        unique_tax = []\n        tax_dict = dict()\n        tax_count = 0\n\n        # Get all unique families/genera/species\n        for group in tax:\n            if not (group in unique_tax):\n                unique_tax.append(group)\n                tax_count += 1\n                tax_dict[group[0:3]] = tax_count\n                if tax_count % 10 == 0:\n                    msg = \"Generating taxonomic groups: \" + str(tax_count) + \" / \" + str(TAX_GROUPS)\n                    sys.stdout.write(msg + \"\\b\" * len(msg))\n        print(\"Generating taxonomic groups: \" + str(TAX_GROUPS) + \" / \" + str(TAX_GROUPS))\n\n\n        # Create species table\n        table = Table(\"species\", delimiter=\",\")\n        table.columns=[(\"species_id\"            ,   (\"pk-int\",)    ),\n                       (\"family\"                ,   (\"char\", )    ),\n                       (\"genus\"                 ,   (\"char\", )    ),\n                       (\"species\"               ,   (\"char\", )    ),\n                       (\"id_level\"              ,   (\"char\", 10)    ),\n                       (\"full_id\"               ,   (\"bool\",)       )]\n\n        data = [','.join([str(tax_dict[group[:3]])] + ['\"%s\"' % g for g in group])\n                for group in unique_tax]\n        table.pk = 'species_id'\n        table.contains_pk = True\n\n        self.engine.table = table\n        self.engine.create_table()\n        self.engine.add_to_table(data)\n\n\n        # Create stems table\n        table = Table(\"stems\", delimiter=\",\", contains_pk=False)\n        table.columns=[(\"stem_id\"               ,   (\"pk-auto\",)    ),\n                       (\"line\"                  ,   (\"int\",)        ),\n                       (\"species_id\"            ,   (\"int\",)        ),\n                       (\"site_code\"             ,   (\"char\", 12)    ),\n                       (\"liana\"                 ,   (\"char\", 10)    ),\n                       (\"stem\"                  ,   (\"double\",)     )]\n        stems = []\n        counts = []\n        for line in lines:\n            try:\n                liana = line[\"liana\"]\n            except KeyError:\n                liana = \"\"\n            species_info = [line[\"line\"],\n                            tax_dict[(line[\"family\"],\n                                      line[\"genus\"],\n                                      line[\"species\"].lower())],\n                            line[\"site\"],\n                            liana\n                            ]\n            try:\n                counts.append([str(value) for value in species_info + [line[\"count\"]]])\n            except KeyError:\n                pass\n\n            for i in line[\"stems\"]:\n                stem = species_info + [i]\n                stems.append([str(value) for value in stem])\n\n        data = [','.join(stem) for stem in stems]\n        self.engine.table = table\n        self.engine.create_table()\n        self.engine.add_to_table(data)\n\n\n        # Create counts table\n        table = Table(\"counts\", delimiter=\",\", contains_pk=False)\n        table.columns=[(\"count_id\"              ,   (\"pk-auto\",)    ),\n                       (\"line\"                  ,   (\"int\",)        ),\n                       (\"species_id\"            ,   (\"int\",)        ),\n                       (\"site_code\"             ,   (\"char\", 12)    ),\n                       (\"liana\"                 ,   (\"char\", 10)    ),\n                       (\"count\"                 ,   (\"double\",)     )]\n        data = [','.join(count) for count in counts]\n        self.engine.table = table\n        self.engine.create_table()\n        self.engine.add_to_table(data)\n\n        return self.engine\n\n\nSCRIPT = main()\n", "path": "scripts/gentry.py"}], "after_files": [{"content": "#retriever\n\"\"\"Retriever script for Alwyn H. Gentry Forest Transect Dataset\n\n\"\"\"\nfrom __future__ import print_function\nfrom builtins import str\nfrom builtins import range\n\nimport os\nimport sys\nimport zipfile\nimport xlrd\nfrom retriever.lib.templates import Script\nfrom retriever.lib.models import Table\nfrom retriever.lib.excel import Excel\n\nVERSION = '1.0.1'\n\nTAX_GROUPS = 9756 #9819\n\n\nclass main(Script):\n    def __init__(self, **kwargs):\n        Script.__init__(self, **kwargs)\n        self.name = \"Alwyn H. Gentry Forest Transect Dataset\"\n        self.shortname = \"Gentry\"\n        self.urls = {\"stems\": \"http://www.mobot.org/mobot/gentry/123/all_Excel.zip\",\n                     \"sites\": \"http://www.ecologicaldata.org/sites/default/files/gentry_sites_data.txt\",\n                     \"species\": \"\",\n                     \"counts\": \"\"}\n        self.tags = [\"Taxon > Plants\", \"Spatial Scale > Global\",\n                     \"Data Type > Observational\"]\n        self.ref = \"http://www.mobot.org/mobot/research/gentry/welcome.shtml\"\n        self.addendum = \"\"\"Researchers who make use of the data in publications are requested to acknowledge Alwyn H. Gentry, the Missouri Botanical Garden, and collectors who assisted Gentry or contributed data for specific sites. It is also requested that a reprint of any publication making use of the Gentry Forest Transect Data be sent to:\n\nBruce E. Ponman\nMissouri Botanical Garden\nP.O. Box 299\nSt. Louis, MO 63166-0299\nU.S.A. \"\"\"\n\n    def download(self, engine=None, debug=False):\n        Script.download(self, engine, debug)\n\n        self.engine.auto_create_table(Table(\"sites\"), url=self.urls[\"sites\"])\n        self.engine.insert_data_from_url(self.urls[\"sites\"])\n\n        self.engine.download_file(self.urls[\"stems\"], \"all_Excel.zip\")\n        local_zip = zipfile.ZipFile(self.engine.format_filename(\"all_Excel.zip\"))\n        filelist = local_zip.namelist()\n        local_zip.close()\n        self.engine.download_files_from_archive(self.urls[\"stems\"], filelist)\n\n        filelist = [os.path.basename(filename) for filename in filelist]\n\n        # Currently all_Excel.zip is missing CURUYUQU.xls\n        # Download it separately and add it to the file list\n        if not self.engine.find_file('CURUYUQU.xls'):\n            self.engine.download_file(\"http://www.mobot.org/mobot/gentry/123/samerica/CURUYUQU.xls\", \"CURUYUQU.xls\")\n            filelist.append('CURUYUQU.xls')\n\n        lines = []\n        tax = []\n        for filename in filelist:\n            print(\"Extracting data from \" + filename + \"...\")\n            book = xlrd.open_workbook(self.engine.format_filename(filename))\n            sh = book.sheet_by_index(0)\n            rows = sh.nrows\n            cn = {'stems': []}\n            n = 0\n            for colnum, c in enumerate(sh.row(0)):\n                if not Excel.empty_cell(c):\n                    cid = Excel.cell_value(c).lower()\n                    # line number column is sometimes named differently\n                    if cid in [\"sub\", \"number\"]:\n                        cid = \"line\"\n                    # the \"number of individuals\" column is named in various\n                    # different ways; they always at least contain \"nd\"\n                    if \"nd\" in cid:\n                        cid = \"count\"\n                    # in QUIAPACA.xls the \"number of individuals\" column is\n                    # misnamed \"STEMDBH\" just like the stems columns, so weep\n                    # for the state of scientific data and then fix manually\n                    if filename == \"QUIAPACA.xls\" and colnum == 13:\n                        cid = \"count\"\n\n                    # if column is a stem, add it to the list of stems;\n                    # otherwise, make note of the column name/number\n                    if \"stem\" in cid or \"dbh\" in cid:\n                        cn[\"stems\"].append(n)\n                    else:\n                        cn[cid] = n\n                n += 1\n            # sometimes, a data file does not contain a liana or count column\n            if not \"liana\" in list(cn.keys()):\n                cn[\"liana\"] = -1\n            if not \"count\" in list(cn.keys()):\n                cn[\"count\"] = -1\n            for i in range(1, rows):\n                row = sh.row(i)\n                cellcount = len(row)\n                # make sure the row is real, not just empty cells\n                if not all(Excel.empty_cell(cell) for cell in row):\n                    try:\n                        this_line = {}\n\n                        def format_value(s):\n                            s = Excel.cell_value(s)\n                            return str(s).title().replace(\"\\\\\", \"/\").replace('\"', '')\n\n                        # get the following information from the appropriate columns\n                        for i in [\"line\", \"family\", \"genus\", \"species\",\n                                  \"liana\", \"count\"]:\n                            if cn[i] > -1:\n                                this_line[i] = format_value(row[cn[i]])\n                                if this_line[i] == '`':\n                                    this_line[i] = 1\n\n                        this_line[\"stems\"] = [Excel.cell_value(row[c])\n                                              for c in cn[\"stems\"]\n                                              if not Excel.empty_cell(row[c])]\n                        this_line[\"site\"] = filename[0:-4]\n\n                        lines.append(this_line)\n\n                        # Check how far the species is identified\n                        full_id = 0\n                        if len(this_line[\"species\"]) < 3:\n                            if len(this_line[\"genus\"]) < 3:\n                                id_level = \"family\"\n                            else:\n                                id_level = \"genus\"\n                        else:\n                            id_level = \"species\"\n                            full_id = 1\n                        tax.append((this_line[\"family\"],\n                                    this_line[\"genus\"],\n                                    this_line[\"species\"].lower().replace('\\\\', '').replace('\"', ''),\n                                    id_level,\n                                    str(full_id)))\n                    except:\n                        raise\n                        pass\n\n        tax = sorted(tax, key=lambda group: group[0] + \" \" + group[1] + \" \" + group[2])\n        unique_tax = []\n        tax_dict = dict()\n        tax_count = 0\n\n        # Get all unique families/genera/species\n        for group in tax:\n            if not (group in unique_tax):\n                unique_tax.append(group)\n                tax_count += 1\n                tax_dict[group[0:3]] = tax_count\n                if tax_count % 10 == 0:\n                    msg = \"Generating taxonomic groups: \" + str(tax_count) + \" / \" + str(TAX_GROUPS)\n                    sys.stdout.write(msg + \"\\b\" * len(msg))\n        print(\"Generating taxonomic groups: \" + str(TAX_GROUPS) + \" / \" + str(TAX_GROUPS))\n\n\n        # Create species table\n        table = Table(\"species\", delimiter=\",\")\n        table.columns=[(\"species_id\"            ,   (\"pk-int\",)    ),\n                       (\"family\"                ,   (\"char\", )    ),\n                       (\"genus\"                 ,   (\"char\", )    ),\n                       (\"species\"               ,   (\"char\", )    ),\n                       (\"id_level\"              ,   (\"char\", 10)    ),\n                       (\"full_id\"               ,   (\"bool\",)       )]\n\n        data = [','.join([str(tax_dict[group[:3]])] + ['\"%s\"' % g for g in group])\n                for group in unique_tax]\n        table.pk = 'species_id'\n        table.contains_pk = True\n\n        self.engine.table = table\n        self.engine.create_table()\n        self.engine.add_to_table(data)\n\n\n        # Create stems table\n        table = Table(\"stems\", delimiter=\",\", contains_pk=False)\n        table.columns=[(\"stem_id\"               ,   (\"pk-auto\",)    ),\n                       (\"line\"                  ,   (\"int\",)        ),\n                       (\"species_id\"            ,   (\"int\",)        ),\n                       (\"site_code\"             ,   (\"char\", 12)    ),\n                       (\"liana\"                 ,   (\"char\", 10)    ),\n                       (\"stem\"                  ,   (\"double\",)     )]\n        stems = []\n        counts = []\n        for line in lines:\n            try:\n                liana = line[\"liana\"]\n            except KeyError:\n                liana = \"\"\n            species_info = [line[\"line\"],\n                            tax_dict[(line[\"family\"],\n                                      line[\"genus\"],\n                                      line[\"species\"].lower())],\n                            line[\"site\"],\n                            liana\n                            ]\n            try:\n                counts.append([str(value) for value in species_info + [line[\"count\"]]])\n            except KeyError:\n                pass\n\n            for i in line[\"stems\"]:\n                stem = species_info + [i]\n                stems.append([str(value) for value in stem])\n\n        data = [','.join(stem) for stem in stems]\n        self.engine.table = table\n        self.engine.create_table()\n        self.engine.add_to_table(data)\n\n\n        # Create counts table\n        table = Table(\"counts\", delimiter=\",\", contains_pk=False)\n        table.columns=[(\"count_id\"              ,   (\"pk-auto\",)    ),\n                       (\"line\"                  ,   (\"int\",)        ),\n                       (\"species_id\"            ,   (\"int\",)        ),\n                       (\"site_code\"             ,   (\"char\", 12)    ),\n                       (\"liana\"                 ,   (\"char\", 10)    ),\n                       (\"count\"                 ,   (\"double\",)     )]\n        data = [','.join(count) for count in counts]\n        self.engine.table = table\n        self.engine.create_table()\n        self.engine.add_to_table(data)\n\n        return self.engine\n\n\nSCRIPT = main()\n", "path": "scripts/gentry.py"}]}