We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Gentry site_code incorrectly includes directory name
At some point a behavior changed that caused site names extracted from the file names in gentry-forest-transects to including the directory in which they were located. This leads to site_code values like `africa/NDAKANNI`, which isn't what we want.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scripts/gentry_forest_transects.py`
Content:
```
1 #retriever
2 """Retriever script for Alwyn H. Gentry Forest Transect Dataset
3 
4 """
5 from __future__ import print_function
6 # from __future__ import unicode_literals
7 from builtins import str
8 from builtins import range
9 
10 import os
11 import sys
12 import zipfile
13 import xlrd
14 from retriever.lib.templates import Script
15 from retriever.lib.models import Table
16 from retriever.lib.excel import Excel
17 from pkg_resources import parse_version
18 try:
19     from retriever.lib.defaults import VERSION
20 except ImportError:
21     from retriever import VERSION
22 
23 TAX_GROUPS = 9756  # 9819
24 
25 
26 class main(Script):
27     def __init__(self, **kwargs):
28         Script.__init__(self, **kwargs)
29         self.title = "Alwyn H. Gentry Forest Transect Dataset"
30         self.name = "gentry-forest-transects"
31         self.retriever_minimum_version = '2.0.dev'
32         self.version = '1.4.2'
33         self.urls = {"stems": "http://www.mobot.org/mobot/gentry/123/all_Excel.zip",
34                      "sites": "https://ndownloader.figshare.com/files/5515373",
35                      "species": "",
36                      "counts": ""}
37         self.keywords = ["plants", "global-scale", "observational"]
38         self.ref = "http://www.mobot.org/mobot/research/gentry/welcome.shtml"
39         self.citation = "Phillips, O. and Miller, J.S., 2002. Global patterns of plant diversity: Alwyn H. Gentry's forest transect data set. Missouri Botanical Press."
40         self.addendum = """Researchers who make use of the data in publications are requested to acknowledge Alwyn H. Gentry, the Missouri Botanical Garden, and collectors who assisted Gentry or contributed data for specific sites. It is also requested that a reprint of any publication making use of the Gentry Forest Transect Data be sent to:
41 
42 Bruce E. Ponman
43 Missouri Botanical Garden
44 P.O. Box 299
45 St. Louis, MO 63166-0299
46 U.S.A. """
47 
48         if parse_version(VERSION) <= parse_version("2.0.0"):
49             self.shortname = self.name
50             self.name = self.title
51             self.tags = self.keywords
52 
53     def download(self, engine=None, debug=False):
54         Script.download(self, engine, debug)
55 
56         self.engine.auto_create_table(Table("sites"), url=self.urls["sites"], filename='gentry_sites.csv')
57         self.engine.insert_data_from_url(self.urls["sites"])
58 
59         self.engine.download_file(self.urls["stems"], "all_Excel.zip")
60         local_zip = zipfile.ZipFile(self.engine.format_filename("all_Excel.zip"))
61         filelist = local_zip.namelist()
62         local_zip.close()
63         self.engine.download_files_from_archive(self.urls["stems"], filelist)
64 
65         if parse_version(VERSION).__str__() < parse_version("2.1.dev").__str__():
66             filelist = [os.path.basename(filename) for filename in filelist]
67 
68         # Currently all_Excel.zip is missing CURUYUQU.xls
69         # Download it separately and add it to the file list
70         if not self.engine.find_file('CURUYUQU.xls'):
71             self.engine.download_file("http://www.mobot.org/mobot/gentry/123/samerica/CURUYUQU.xls", "CURUYUQU.xls")
72             filelist.append('CURUYUQU.xls')
73 
74         lines = []
75         tax = []
76         for filename in filelist:
77             print("Extracting data from " + filename + "...")
78             book = xlrd.open_workbook(self.engine.format_filename(filename))
79             sh = book.sheet_by_index(0)
80             rows = sh.nrows
81             cn = {'stems': []}
82             n = 0
83             for colnum, c in enumerate(sh.row(0)):
84                 if not Excel.empty_cell(c):
85                     cid = c.value.lower().strip()
86                     # line number column is sometimes named differently
87                     if cid in ["sub", "number"]:
88                         cid = "line"
89                     # the "number of individuals" column is named in various
90                     # different ways; they always at least contain "nd"
91                     if "nd" in cid:
92                         cid = "count"
93                     # in QUIAPACA.xls the "number of individuals" column is
94                     # misnamed "STEMDBH" just like the stems columns, so weep
95                     # for the state of scientific data and then fix manually
96                     if filename == "QUIAPACA.xls" and colnum == 13:
97                         cid = "count"
98 
99                     # if column is a stem, add it to the list of stems;
100                     # otherwise, make note of the column name/number
101                     if "stem" in cid or "dbh" in cid:
102                         cn["stems"].append(n)
103                     else:
104                         cn[cid] = n
105                 n += 1
106             # sometimes, a data file does not contain a liana or count column
107             if not "liana" in list(cn.keys()):
108                 cn["liana"] = -1
109             if not "count" in list(cn.keys()):
110                 cn["count"] = -1
111             for i in range(1, rows):
112                 row = sh.row(i)
113                 cellcount = len(row)
114                 # make sure the row is real, not just empty cells
115                 if not all(Excel.empty_cell(cell) for cell in row):
116                     try:
117                         this_line = {}
118 
119                         # get the following information from the appropriate columns
120                         for i in ["line", "family", "genus", "species",
121                                   "liana", "count"]:
122                             if cn[i] > -1:
123                                 if row[cn[i]].ctype != 2:
124                                     # if the cell type(ctype) is not a number
125                                     this_line[i] = row[cn[i]].value.lower().strip().replace("\\", "/").replace('"', '')
126                                 else:
127                                     this_line[i] = row[cn[i]].value
128                                 if this_line[i] == '`':
129                                     this_line[i] = 1
130                         this_line["stems"] = [row[c]
131                                               for c in cn["stems"]
132                                               if not Excel.empty_cell(row[c])]
133                         this_line["site"] = filename[0:-4]
134 
135                         # Manually correct CEDRAL data, which has a single line
136                         # that is shifted by one to the left starting at Liana
137                         if this_line["site"] == "CEDRAL" and type(this_line["liana"]) == float:
138                             this_line["liana"] = ""
139                             this_line["count"] = 3
140                             this_line["stems"] = [2.5, 2.5, 30, 18, 25]
141 
142                         lines.append(this_line)
143 
144                         # Check how far the species is identified
145                         full_id = 0
146                         if len(this_line["species"]) < 3:
147                             if len(this_line["genus"]) < 3:
148                                 id_level = "family"
149                             else:
150                                 id_level = "genus"
151                         else:
152                             id_level = "species"
153                             full_id = 1
154                         tax.append((this_line["family"],
155                                     this_line["genus"],
156                                     this_line["species"],
157                                     id_level,
158                                     str(full_id)))
159                     except:
160                         raise
161                         pass
162 
163         tax = sorted(tax, key=lambda group: group[0] + " " + group[1] + " " + group[2])
164         unique_tax = []
165         tax_dict = {}
166         tax_count = 0
167 
168         # Get all unique families/genera/species
169         print("\n")
170         for group in tax:
171             if not (group in unique_tax):
172                 unique_tax.append(group)
173                 tax_count += 1
174                 tax_dict[group[0:3]] = tax_count
175                 if tax_count % 10 == 0:
176                     msg = "Generating taxonomic groups: " + str(tax_count) + " / " + str(TAX_GROUPS)
177                     sys.stdout.flush()
178                     sys.stdout.write(msg + "\b" * len(msg))
179         print("\n")
180         # Create species table
181         table = Table("species", delimiter=",")
182         table.columns=[("species_id"            ,   ("pk-int",)    ),
183                        ("family"                ,   ("char", )    ),
184                        ("genus"                 ,   ("char", )    ),
185                        ("species"               ,   ("char", )    ),
186                        ("id_level"              ,   ("char", 10)    ),
187                        ("full_id"               ,   ("int",)       )]
188 
189         data = [[str(tax_dict[group[:3]])] + ['"%s"' % g for g in group]
190                 for group in unique_tax]
191         table.pk = 'species_id'
192         table.contains_pk = True
193 
194         self.engine.table = table
195         self.engine.create_table()
196         self.engine.add_to_table(data)
197 
198         # Create stems table
199         table = Table("stems", delimiter=",")
200         table.columns=[("stem_id"               ,   ("pk-auto",)    ),
201                        ("line"                  ,   ("int",)        ),
202                        ("species_id"            ,   ("int",)        ),
203                        ("site_code"             ,   ("char", 12)    ),
204                        ("liana"                 ,   ("char", 10)    ),
205                        ("stem"                  ,   ("double",)     )]
206         stems = []
207         counts = []
208         for line in lines:
209             try:
210                 liana = line["liana"]
211             except KeyError:
212                 liana = ""
213             species_info = [line["line"],
214                             tax_dict[(line["family"],
215                                       line["genus"],
216                                       line["species"])],
217                             line["site"],
218                             liana
219                             ]
220             try:
221                 counts.append([value for value in species_info + [line["count"]]])
222             except KeyError:
223                 pass
224 
225             for i in line["stems"]:
226                 stem = species_info + [str(i)]
227                 stems.append(stem)
228 
229         self.engine.table = table
230         self.engine.create_table()
231         self.engine.add_to_table(stems)
232 
233         # Create counts table
234         table = Table("counts", delimiter=",", contains_pk=False)
235         table.columns=[("count_id"              ,   ("pk-auto",)    ),
236                        ("line"                  ,   ("int",)        ),
237                        ("species_id"            ,   ("int",)        ),
238                        ("site_code"             ,   ("char", 12)    ),
239                        ("liana"                 ,   ("char", 10)    ),
240                        ("count"                 ,   ("double",)     )]
241         self.engine.table = table
242         self.engine.create_table()
243         self.engine.add_to_table(counts)
244 
245         return self.engine
246 SCRIPT = main()
247
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scripts/gentry_forest_transects.py b/scripts/gentry_forest_transects.py
--- a/scripts/gentry_forest_transects.py
+++ b/scripts/gentry_forest_transects.py
@@ -1,9 +1,6 @@
 #retriever
-"""Retriever script for Alwyn H. Gentry Forest Transect Dataset
+"""Retriever script for Alwyn H. Gentry Forest Transect Dataset"""
 
-"""
-from __future__ import print_function
-# from __future__ import unicode_literals
 from builtins import str
 from builtins import range
 
@@ -29,7 +26,7 @@
         self.title = "Alwyn H. Gentry Forest Transect Dataset"
         self.name = "gentry-forest-transects"
         self.retriever_minimum_version = '2.0.dev'
-        self.version = '1.4.2'
+        self.version = '1.4.3'
         self.urls = {"stems": "http://www.mobot.org/mobot/gentry/123/all_Excel.zip",
                      "sites": "https://ndownloader.figshare.com/files/5515373",
                      "species": "",
@@ -74,7 +71,6 @@
         lines = []
         tax = []
         for filename in filelist:
-            print("Extracting data from " + filename + "...")
             book = xlrd.open_workbook(self.engine.format_filename(filename))
             sh = book.sheet_by_index(0)
             rows = sh.nrows
@@ -130,7 +126,8 @@
                         this_line["stems"] = [row[c]
                                               for c in cn["stems"]
                                               if not Excel.empty_cell(row[c])]
-                        this_line["site"] = filename[0:-4]
+                        site_code, _ = os.path.splitext(os.path.basename(filename))
+                        this_line["site"] = site_code
 
                         # Manually correct CEDRAL data, which has a single line
                         # that is shifted by one to the left starting at Liana
@@ -166,17 +163,11 @@
         tax_count = 0
 
         # Get all unique families/genera/species
-        print("\n")
         for group in tax:
             if not (group in unique_tax):
                 unique_tax.append(group)
                 tax_count += 1
                 tax_dict[group[0:3]] = tax_count
-                if tax_count % 10 == 0:
-                    msg = "Generating taxonomic groups: " + str(tax_count) + " / " + str(TAX_GROUPS)
-                    sys.stdout.flush()
-                    sys.stdout.write(msg + "\b" * len(msg))
-        print("\n")
         # Create species table
         table = Table("species", delimiter=",")
         table.columns=[("species_id"            ,   ("pk-int",)    ),

{"golden_diff": "diff --git a/scripts/gentry_forest_transects.py b/scripts/gentry_forest_transects.py\n--- a/scripts/gentry_forest_transects.py\n+++ b/scripts/gentry_forest_transects.py\n@@ -1,9 +1,6 @@\n #retriever\n-\"\"\"Retriever script for Alwyn H. Gentry Forest Transect Dataset\n+\"\"\"Retriever script for Alwyn H. Gentry Forest Transect Dataset\"\"\"\n \n-\"\"\"\n-from __future__ import print_function\n-# from __future__ import unicode_literals\n from builtins import str\n from builtins import range\n \n@@ -29,7 +26,7 @@\n         self.title = \"Alwyn H. Gentry Forest Transect Dataset\"\n         self.name = \"gentry-forest-transects\"\n         self.retriever_minimum_version = '2.0.dev'\n-        self.version = '1.4.2'\n+        self.version = '1.4.3'\n         self.urls = {\"stems\": \"http://www.mobot.org/mobot/gentry/123/all_Excel.zip\",\n                      \"sites\": \"https://ndownloader.figshare.com/files/5515373\",\n                      \"species\": \"\",\n@@ -74,7 +71,6 @@\n         lines = []\n         tax = []\n         for filename in filelist:\n-            print(\"Extracting data from \" + filename + \"...\")\n             book = xlrd.open_workbook(self.engine.format_filename(filename))\n             sh = book.sheet_by_index(0)\n             rows = sh.nrows\n@@ -130,7 +126,8 @@\n                         this_line[\"stems\"] = [row[c]\n                                               for c in cn[\"stems\"]\n                                               if not Excel.empty_cell(row[c])]\n-                        this_line[\"site\"] = filename[0:-4]\n+                        site_code, _ = os.path.splitext(os.path.basename(filename))\n+                        this_line[\"site\"] = site_code\n \n                         # Manually correct CEDRAL data, which has a single line\n                         # that is shifted by one to the left starting at Liana\n@@ -166,17 +163,11 @@\n         tax_count = 0\n \n         # Get all unique families/genera/species\n-        print(\"\\n\")\n         for group in tax:\n             if not (group in unique_tax):\n                 unique_tax.append(group)\n                 tax_count += 1\n                 tax_dict[group[0:3]] = tax_count\n-                if tax_count % 10 == 0:\n-                    msg = \"Generating taxonomic groups: \" + str(tax_count) + \" / \" + str(TAX_GROUPS)\n-                    sys.stdout.flush()\n-                    sys.stdout.write(msg + \"\\b\" * len(msg))\n-        print(\"\\n\")\n         # Create species table\n         table = Table(\"species\", delimiter=\",\")\n         table.columns=[(\"species_id\"            ,   (\"pk-int\",)    ),\n", "issue": "Gentry site_code incorrectly includes directory name\nAt some point a behavior changed that caused site names extracted from the file names in gentry-forest-transects to including the directory in which they were located. This leads to site_code values like `africa/NDAKANNI`, which isn't what we want.\n", "before_files": [{"content": "#retriever\n\"\"\"Retriever script for Alwyn H. Gentry Forest Transect Dataset\n\n\"\"\"\nfrom __future__ import print_function\n# from __future__ import unicode_literals\nfrom builtins import str\nfrom builtins import range\n\nimport os\nimport sys\nimport zipfile\nimport xlrd\nfrom retriever.lib.templates import Script\nfrom retriever.lib.models import Table\nfrom retriever.lib.excel import Excel\nfrom pkg_resources import parse_version\ntry:\n    from retriever.lib.defaults import VERSION\nexcept ImportError:\n    from retriever import VERSION\n\nTAX_GROUPS = 9756  # 9819\n\n\nclass main(Script):\n    def __init__(self, **kwargs):\n        Script.__init__(self, **kwargs)\n        self.title = \"Alwyn H. Gentry Forest Transect Dataset\"\n        self.name = \"gentry-forest-transects\"\n        self.retriever_minimum_version = '2.0.dev'\n        self.version = '1.4.2'\n        self.urls = {\"stems\": \"http://www.mobot.org/mobot/gentry/123/all_Excel.zip\",\n                     \"sites\": \"https://ndownloader.figshare.com/files/5515373\",\n                     \"species\": \"\",\n                     \"counts\": \"\"}\n        self.keywords = [\"plants\", \"global-scale\", \"observational\"]\n        self.ref = \"http://www.mobot.org/mobot/research/gentry/welcome.shtml\"\n        self.citation = \"Phillips, O. and Miller, J.S., 2002. Global patterns of plant diversity: Alwyn H. Gentry's forest transect data set. Missouri Botanical Press.\"\n        self.addendum = \"\"\"Researchers who make use of the data in publications are requested to acknowledge Alwyn H. Gentry, the Missouri Botanical Garden, and collectors who assisted Gentry or contributed data for specific sites. It is also requested that a reprint of any publication making use of the Gentry Forest Transect Data be sent to:\n\nBruce E. Ponman\nMissouri Botanical Garden\nP.O. Box 299\nSt. Louis, MO 63166-0299\nU.S.A. \"\"\"\n\n        if parse_version(VERSION) <= parse_version(\"2.0.0\"):\n            self.shortname = self.name\n            self.name = self.title\n            self.tags = self.keywords\n\n    def download(self, engine=None, debug=False):\n        Script.download(self, engine, debug)\n\n        self.engine.auto_create_table(Table(\"sites\"), url=self.urls[\"sites\"], filename='gentry_sites.csv')\n        self.engine.insert_data_from_url(self.urls[\"sites\"])\n\n        self.engine.download_file(self.urls[\"stems\"], \"all_Excel.zip\")\n        local_zip = zipfile.ZipFile(self.engine.format_filename(\"all_Excel.zip\"))\n        filelist = local_zip.namelist()\n        local_zip.close()\n        self.engine.download_files_from_archive(self.urls[\"stems\"], filelist)\n\n        if parse_version(VERSION).__str__() < parse_version(\"2.1.dev\").__str__():\n            filelist = [os.path.basename(filename) for filename in filelist]\n\n        # Currently all_Excel.zip is missing CURUYUQU.xls\n        # Download it separately and add it to the file list\n        if not self.engine.find_file('CURUYUQU.xls'):\n            self.engine.download_file(\"http://www.mobot.org/mobot/gentry/123/samerica/CURUYUQU.xls\", \"CURUYUQU.xls\")\n            filelist.append('CURUYUQU.xls')\n\n        lines = []\n        tax = []\n        for filename in filelist:\n            print(\"Extracting data from \" + filename + \"...\")\n            book = xlrd.open_workbook(self.engine.format_filename(filename))\n            sh = book.sheet_by_index(0)\n            rows = sh.nrows\n            cn = {'stems': []}\n            n = 0\n            for colnum, c in enumerate(sh.row(0)):\n                if not Excel.empty_cell(c):\n                    cid = c.value.lower().strip()\n                    # line number column is sometimes named differently\n                    if cid in [\"sub\", \"number\"]:\n                        cid = \"line\"\n                    # the \"number of individuals\" column is named in various\n                    # different ways; they always at least contain \"nd\"\n                    if \"nd\" in cid:\n                        cid = \"count\"\n                    # in QUIAPACA.xls the \"number of individuals\" column is\n                    # misnamed \"STEMDBH\" just like the stems columns, so weep\n                    # for the state of scientific data and then fix manually\n                    if filename == \"QUIAPACA.xls\" and colnum == 13:\n                        cid = \"count\"\n\n                    # if column is a stem, add it to the list of stems;\n                    # otherwise, make note of the column name/number\n                    if \"stem\" in cid or \"dbh\" in cid:\n                        cn[\"stems\"].append(n)\n                    else:\n                        cn[cid] = n\n                n += 1\n            # sometimes, a data file does not contain a liana or count column\n            if not \"liana\" in list(cn.keys()):\n                cn[\"liana\"] = -1\n            if not \"count\" in list(cn.keys()):\n                cn[\"count\"] = -1\n            for i in range(1, rows):\n                row = sh.row(i)\n                cellcount = len(row)\n                # make sure the row is real, not just empty cells\n                if not all(Excel.empty_cell(cell) for cell in row):\n                    try:\n                        this_line = {}\n\n                        # get the following information from the appropriate columns\n                        for i in [\"line\", \"family\", \"genus\", \"species\",\n                                  \"liana\", \"count\"]:\n                            if cn[i] > -1:\n                                if row[cn[i]].ctype != 2:\n                                    # if the cell type(ctype) is not a number\n                                    this_line[i] = row[cn[i]].value.lower().strip().replace(\"\\\\\", \"/\").replace('\"', '')\n                                else:\n                                    this_line[i] = row[cn[i]].value\n                                if this_line[i] == '`':\n                                    this_line[i] = 1\n                        this_line[\"stems\"] = [row[c]\n                                              for c in cn[\"stems\"]\n                                              if not Excel.empty_cell(row[c])]\n                        this_line[\"site\"] = filename[0:-4]\n\n                        # Manually correct CEDRAL data, which has a single line\n                        # that is shifted by one to the left starting at Liana\n                        if this_line[\"site\"] == \"CEDRAL\" and type(this_line[\"liana\"]) == float:\n                            this_line[\"liana\"] = \"\"\n                            this_line[\"count\"] = 3\n                            this_line[\"stems\"] = [2.5, 2.5, 30, 18, 25]\n\n                        lines.append(this_line)\n\n                        # Check how far the species is identified\n                        full_id = 0\n                        if len(this_line[\"species\"]) < 3:\n                            if len(this_line[\"genus\"]) < 3:\n                                id_level = \"family\"\n                            else:\n                                id_level = \"genus\"\n                        else:\n                            id_level = \"species\"\n                            full_id = 1\n                        tax.append((this_line[\"family\"],\n                                    this_line[\"genus\"],\n                                    this_line[\"species\"],\n                                    id_level,\n                                    str(full_id)))\n                    except:\n                        raise\n                        pass\n\n        tax = sorted(tax, key=lambda group: group[0] + \" \" + group[1] + \" \" + group[2])\n        unique_tax = []\n        tax_dict = {}\n        tax_count = 0\n\n        # Get all unique families/genera/species\n        print(\"\\n\")\n        for group in tax:\n            if not (group in unique_tax):\n                unique_tax.append(group)\n                tax_count += 1\n                tax_dict[group[0:3]] = tax_count\n                if tax_count % 10 == 0:\n                    msg = \"Generating taxonomic groups: \" + str(tax_count) + \" / \" + str(TAX_GROUPS)\n                    sys.stdout.flush()\n                    sys.stdout.write(msg + \"\\b\" * len(msg))\n        print(\"\\n\")\n        # Create species table\n        table = Table(\"species\", delimiter=\",\")\n        table.columns=[(\"species_id\"            ,   (\"pk-int\",)    ),\n                       (\"family\"                ,   (\"char\", )    ),\n                       (\"genus\"                 ,   (\"char\", )    ),\n                       (\"species\"               ,   (\"char\", )    ),\n                       (\"id_level\"              ,   (\"char\", 10)    ),\n                       (\"full_id\"               ,   (\"int\",)       )]\n\n        data = [[str(tax_dict[group[:3]])] + ['\"%s\"' % g for g in group]\n                for group in unique_tax]\n        table.pk = 'species_id'\n        table.contains_pk = True\n\n        self.engine.table = table\n        self.engine.create_table()\n        self.engine.add_to_table(data)\n\n        # Create stems table\n        table = Table(\"stems\", delimiter=\",\")\n        table.columns=[(\"stem_id\"               ,   (\"pk-auto\",)    ),\n                       (\"line\"                  ,   (\"int\",)        ),\n                       (\"species_id\"            ,   (\"int\",)        ),\n                       (\"site_code\"             ,   (\"char\", 12)    ),\n                       (\"liana\"                 ,   (\"char\", 10)    ),\n                       (\"stem\"                  ,   (\"double\",)     )]\n        stems = []\n        counts = []\n        for line in lines:\n            try:\n                liana = line[\"liana\"]\n            except KeyError:\n                liana = \"\"\n            species_info = [line[\"line\"],\n                            tax_dict[(line[\"family\"],\n                                      line[\"genus\"],\n                                      line[\"species\"])],\n                            line[\"site\"],\n                            liana\n                            ]\n            try:\n                counts.append([value for value in species_info + [line[\"count\"]]])\n            except KeyError:\n                pass\n\n            for i in line[\"stems\"]:\n                stem = species_info + [str(i)]\n                stems.append(stem)\n\n        self.engine.table = table\n        self.engine.create_table()\n        self.engine.add_to_table(stems)\n\n        # Create counts table\n        table = Table(\"counts\", delimiter=\",\", contains_pk=False)\n        table.columns=[(\"count_id\"              ,   (\"pk-auto\",)    ),\n                       (\"line\"                  ,   (\"int\",)        ),\n                       (\"species_id\"            ,   (\"int\",)        ),\n                       (\"site_code\"             ,   (\"char\", 12)    ),\n                       (\"liana\"                 ,   (\"char\", 10)    ),\n                       (\"count\"                 ,   (\"double\",)     )]\n        self.engine.table = table\n        self.engine.create_table()\n        self.engine.add_to_table(counts)\n\n        return self.engine\nSCRIPT = main()\n", "path": "scripts/gentry_forest_transects.py"}], "after_files": [{"content": "#retriever\n\"\"\"Retriever script for Alwyn H. Gentry Forest Transect Dataset\"\"\"\n\nfrom builtins import str\nfrom builtins import range\n\nimport os\nimport sys\nimport zipfile\nimport xlrd\nfrom retriever.lib.templates import Script\nfrom retriever.lib.models import Table\nfrom retriever.lib.excel import Excel\nfrom pkg_resources import parse_version\ntry:\n    from retriever.lib.defaults import VERSION\nexcept ImportError:\n    from retriever import VERSION\n\nTAX_GROUPS = 9756  # 9819\n\n\nclass main(Script):\n    def __init__(self, **kwargs):\n        Script.__init__(self, **kwargs)\n        self.title = \"Alwyn H. Gentry Forest Transect Dataset\"\n        self.name = \"gentry-forest-transects\"\n        self.retriever_minimum_version = '2.0.dev'\n        self.version = '1.4.3'\n        self.urls = {\"stems\": \"http://www.mobot.org/mobot/gentry/123/all_Excel.zip\",\n                     \"sites\": \"https://ndownloader.figshare.com/files/5515373\",\n                     \"species\": \"\",\n                     \"counts\": \"\"}\n        self.keywords = [\"plants\", \"global-scale\", \"observational\"]\n        self.ref = \"http://www.mobot.org/mobot/research/gentry/welcome.shtml\"\n        self.citation = \"Phillips, O. and Miller, J.S., 2002. Global patterns of plant diversity: Alwyn H. Gentry's forest transect data set. Missouri Botanical Press.\"\n        self.addendum = \"\"\"Researchers who make use of the data in publications are requested to acknowledge Alwyn H. Gentry, the Missouri Botanical Garden, and collectors who assisted Gentry or contributed data for specific sites. It is also requested that a reprint of any publication making use of the Gentry Forest Transect Data be sent to:\n\nBruce E. Ponman\nMissouri Botanical Garden\nP.O. Box 299\nSt. Louis, MO 63166-0299\nU.S.A. \"\"\"\n\n        if parse_version(VERSION) <= parse_version(\"2.0.0\"):\n            self.shortname = self.name\n            self.name = self.title\n            self.tags = self.keywords\n\n    def download(self, engine=None, debug=False):\n        Script.download(self, engine, debug)\n\n        self.engine.auto_create_table(Table(\"sites\"), url=self.urls[\"sites\"], filename='gentry_sites.csv')\n        self.engine.insert_data_from_url(self.urls[\"sites\"])\n\n        self.engine.download_file(self.urls[\"stems\"], \"all_Excel.zip\")\n        local_zip = zipfile.ZipFile(self.engine.format_filename(\"all_Excel.zip\"))\n        filelist = local_zip.namelist()\n        local_zip.close()\n        self.engine.download_files_from_archive(self.urls[\"stems\"], filelist)\n\n        if parse_version(VERSION).__str__() < parse_version(\"2.1.dev\").__str__():\n            filelist = [os.path.basename(filename) for filename in filelist]\n\n        # Currently all_Excel.zip is missing CURUYUQU.xls\n        # Download it separately and add it to the file list\n        if not self.engine.find_file('CURUYUQU.xls'):\n            self.engine.download_file(\"http://www.mobot.org/mobot/gentry/123/samerica/CURUYUQU.xls\", \"CURUYUQU.xls\")\n            filelist.append('CURUYUQU.xls')\n\n        lines = []\n        tax = []\n        for filename in filelist:\n            book = xlrd.open_workbook(self.engine.format_filename(filename))\n            sh = book.sheet_by_index(0)\n            rows = sh.nrows\n            cn = {'stems': []}\n            n = 0\n            for colnum, c in enumerate(sh.row(0)):\n                if not Excel.empty_cell(c):\n                    cid = c.value.lower().strip()\n                    # line number column is sometimes named differently\n                    if cid in [\"sub\", \"number\"]:\n                        cid = \"line\"\n                    # the \"number of individuals\" column is named in various\n                    # different ways; they always at least contain \"nd\"\n                    if \"nd\" in cid:\n                        cid = \"count\"\n                    # in QUIAPACA.xls the \"number of individuals\" column is\n                    # misnamed \"STEMDBH\" just like the stems columns, so weep\n                    # for the state of scientific data and then fix manually\n                    if filename == \"QUIAPACA.xls\" and colnum == 13:\n                        cid = \"count\"\n\n                    # if column is a stem, add it to the list of stems;\n                    # otherwise, make note of the column name/number\n                    if \"stem\" in cid or \"dbh\" in cid:\n                        cn[\"stems\"].append(n)\n                    else:\n                        cn[cid] = n\n                n += 1\n            # sometimes, a data file does not contain a liana or count column\n            if not \"liana\" in list(cn.keys()):\n                cn[\"liana\"] = -1\n            if not \"count\" in list(cn.keys()):\n                cn[\"count\"] = -1\n            for i in range(1, rows):\n                row = sh.row(i)\n                cellcount = len(row)\n                # make sure the row is real, not just empty cells\n                if not all(Excel.empty_cell(cell) for cell in row):\n                    try:\n                        this_line = {}\n\n                        # get the following information from the appropriate columns\n                        for i in [\"line\", \"family\", \"genus\", \"species\",\n                                  \"liana\", \"count\"]:\n                            if cn[i] > -1:\n                                if row[cn[i]].ctype != 2:\n                                    # if the cell type(ctype) is not a number\n                                    this_line[i] = row[cn[i]].value.lower().strip().replace(\"\\\\\", \"/\").replace('\"', '')\n                                else:\n                                    this_line[i] = row[cn[i]].value\n                                if this_line[i] == '`':\n                                    this_line[i] = 1\n                        this_line[\"stems\"] = [row[c]\n                                              for c in cn[\"stems\"]\n                                              if not Excel.empty_cell(row[c])]\n                        site_code, _ = os.path.splitext(os.path.basename(filename))\n                        this_line[\"site\"] = site_code\n\n                        # Manually correct CEDRAL data, which has a single line\n                        # that is shifted by one to the left starting at Liana\n                        if this_line[\"site\"] == \"CEDRAL\" and type(this_line[\"liana\"]) == float:\n                            this_line[\"liana\"] = \"\"\n                            this_line[\"count\"] = 3\n                            this_line[\"stems\"] = [2.5, 2.5, 30, 18, 25]\n\n                        lines.append(this_line)\n\n                        # Check how far the species is identified\n                        full_id = 0\n                        if len(this_line[\"species\"]) < 3:\n                            if len(this_line[\"genus\"]) < 3:\n                                id_level = \"family\"\n                            else:\n                                id_level = \"genus\"\n                        else:\n                            id_level = \"species\"\n                            full_id = 1\n                        tax.append((this_line[\"family\"],\n                                    this_line[\"genus\"],\n                                    this_line[\"species\"],\n                                    id_level,\n                                    str(full_id)))\n                    except:\n                        raise\n                        pass\n\n        tax = sorted(tax, key=lambda group: group[0] + \" \" + group[1] + \" \" + group[2])\n        unique_tax = []\n        tax_dict = {}\n        tax_count = 0\n\n        # Get all unique families/genera/species\n        for group in tax:\n            if not (group in unique_tax):\n                unique_tax.append(group)\n                tax_count += 1\n                tax_dict[group[0:3]] = tax_count\n        # Create species table\n        table = Table(\"species\", delimiter=\",\")\n        table.columns=[(\"species_id\"            ,   (\"pk-int\",)    ),\n                       (\"family\"                ,   (\"char\", )    ),\n                       (\"genus\"                 ,   (\"char\", )    ),\n                       (\"species\"               ,   (\"char\", )    ),\n                       (\"id_level\"              ,   (\"char\", 10)    ),\n                       (\"full_id\"               ,   (\"int\",)       )]\n\n        data = [[str(tax_dict[group[:3]])] + ['\"%s\"' % g for g in group]\n                for group in unique_tax]\n        table.pk = 'species_id'\n        table.contains_pk = True\n\n        self.engine.table = table\n        self.engine.create_table()\n        self.engine.add_to_table(data)\n\n        # Create stems table\n        table = Table(\"stems\", delimiter=\",\")\n        table.columns=[(\"stem_id\"               ,   (\"pk-auto\",)    ),\n                       (\"line\"                  ,   (\"int\",)        ),\n                       (\"species_id\"            ,   (\"int\",)        ),\n                       (\"site_code\"             ,   (\"char\", 12)    ),\n                       (\"liana\"                 ,   (\"char\", 10)    ),\n                       (\"stem\"                  ,   (\"double\",)     )]\n        stems = []\n        counts = []\n        for line in lines:\n            try:\n                liana = line[\"liana\"]\n            except KeyError:\n                liana = \"\"\n            species_info = [line[\"line\"],\n                            tax_dict[(line[\"family\"],\n                                      line[\"genus\"],\n                                      line[\"species\"])],\n                            line[\"site\"],\n                            liana\n                            ]\n            try:\n                counts.append([value for value in species_info + [line[\"count\"]]])\n            except KeyError:\n                pass\n\n            for i in line[\"stems\"]:\n                stem = species_info + [str(i)]\n                stems.append(stem)\n\n        self.engine.table = table\n        self.engine.create_table()\n        self.engine.add_to_table(stems)\n\n        # Create counts table\n        table = Table(\"counts\", delimiter=\",\", contains_pk=False)\n        table.columns=[(\"count_id\"              ,   (\"pk-auto\",)    ),\n                       (\"line\"                  ,   (\"int\",)        ),\n                       (\"species_id\"            ,   (\"int\",)        ),\n                       (\"site_code\"             ,   (\"char\", 12)    ),\n                       (\"liana\"                 ,   (\"char\", 10)    ),\n                       (\"count\"                 ,   (\"double\",)     )]\n        self.engine.table = table\n        self.engine.create_table()\n        self.engine.add_to_table(counts)\n\n        return self.engine\nSCRIPT = main()\n", "path": "scripts/gentry_forest_transects.py"}]}