We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Hydro missing from DO
![opbeat com-tomorrow-electricity-map-backend-errors-329-](https://user-images.githubusercontent.com/1655848/29333158-11e0dec0-8203-11e7-9e38-a24c2d74e42f.png)


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `parsers/DO.py`
Content:
```
1 #!/usr/bin/env python
2 
3 import arrow
4 from bs4 import BeautifulSoup
5 from collections import defaultdict
6 import itertools
7 from math import isnan
8 import numpy as np
9 from operator import itemgetter
10 import pandas as pd
11 import requests
12 
13 
14 #This parser gets hourly electricity generation data from oc.org.do for the Dominican Republic.
15 #The data is in MWh but since it is updated hourly we can view it as MW.
16 #Solar generation has no data available currently but multiple projects are planned/under construction.
17 
18 url = 'http://184.168.74.190:81/ReportesGraficos/ReportePostdespacho.aspx'
19 
20 total_mapping = {
21                 u'Total T\xe9rmico': 'Thermal',
22                 u'Total E\xf3lico': 'Wind',
23                 u'Total Hidroel\xe9ctrica': 'Hydro',
24                 'Total Generado': 'Generated'
25                 }
26 
27 #Power plant types
28 #http://www.sie.gob.do/images/Estadisticas/MEM/GeneracionDiariaEnero2017/
29 #Reporte_diario_de_generacion_31_enero_2017_merged2.pdf
30 
31 thermal_plants = {
32                  u'AES ANDRES': 'gas',
33                  u'BARAHONA CARBON': 'coal',
34                  u'BERSAL': 'unknown',
35                  u'CEPP 1': 'oil',
36                  u'CEPP 2': 'oil',
37                  u'CESPM 1': 'oil',
38                  u'CESPM 2': 'oil',
39                  u'CESPM 3': 'oil',
40                  u'ESTRELLA DEL MAR 2 CFO': 'oil',
41                  u'ESTRELLA DEL MAR 2 CGN': 'gas',
42                  u'ESTRELLA DEL MAR 2 SFO': 'oil',
43                  u'ESTRELLA DEL MAR 2 SGN': 'gas',
44                  u'HAINA TG': 'oil',
45                  u'INCA KM22': 'oil',
46                  u'ITABO 1': 'coal',
47                  u'ITABO 2': 'coal',
48                  u'LA VEGA': 'oil',
49                  u'LOS MINA 5': 'gas',
50                  u'LOS MINA 6': 'gas',
51                  u'LOS MINA 7': 'gas',
52                  u'LOS OR\xcdGENES POWER PLANT FUEL OIL': 'oil',
53                  u'LOS OR\xcdGENES POWER PLANT GAS NATURAL': 'gas',
54                  u'METALDOM': 'oil',
55                  u'MONTE PLATA SOLAR': 'solar',
56                  u'MONTE RIO': 'oil',
57                  u'PALAMARA': 'oil',
58                  u'PARQUE ENERGETICO LOS MINA CC PARCIAL': 'gas',
59                  u'PARQUE ENERGETICO LOS MINA CC TOTAL': 'gas',
60                  u'PIMENTEL 1': 'oil',
61                  u'PIMENTEL 2': 'oil',
62                  u'PIMENTEL 3': 'oil',
63                  u'QUISQUEYA 1': 'gas',
64                  u'QUISQUEYA 2': 'gas',
65                  u'RIO SAN JUAN': 'oil',
66                  u'SAN FELIPE': 'oil',
67                  u'SAN FELIPE CC': 'gas',
68                  u'SAN FELIPE VAP': 'oil',
69                  u'SAN LORENZO 1': 'gas',
70                  u'SAN PEDRO BIO-ENERGY': 'biomass',
71                  u'SAN PEDRO VAPOR': 'oil',
72                  u'SULTANA DEL ESTE': 'oil'
73                  }
74 
75 
76 def get_data(session = None):
77     """
78     Makes a request to source url.
79     Finds main table and creates a list of all table elements in unicode string format.
80     Returns a list.
81     """
82 
83     data = []
84     s = session or requests.Session()
85     data_req = s.get(url)
86     soup = BeautifulSoup(data_req.content, 'lxml')
87 
88     tbs = soup.find("table", id="PostdespachoUnidadesTermicasGrid_DXMainTable")
89     rows = tbs.find_all("td")
90 
91     for row in rows:
92         num = row.getText().strip()
93         data.append(unicode(num))
94 
95     return data
96 
97 
98 def floater(item):
99     """
100     Attempts to convert any item given to a float.  Returns item if it fails.
101     """
102 
103     try:
104         return float(item)
105     except ValueError:
106         return item
107 
108 
109 def chunker(big_lst):
110     """
111     Breaks a big list into a list of lists.  Removes any list with no data then turns remaining
112     lists into key: value pairs with first element from the list being the key.
113     Returns a dictionary.
114     """
115     
116     chunks = [big_lst[x:x+27] for x in xrange(0, len(big_lst), 27)]
117 
118     #Remove the list if it contains no data.
119     for chunk in chunks:
120         if any(chunk) == True:
121             continue
122         else:
123             chunks.remove(chunk)
124 
125     chunked_list = {words[0]:words[1:] for words in chunks}
126 
127     return chunked_list
128 
129 
130 def data_formatter(data):
131     """
132     Takes data and finds relevant sections.  Formats and breaks data into usable parts.
133     Returns a nested dictionary.
134     """
135 
136     find_thermal_index = data.index(u'GRUPO: T\xe9rmica')
137     find_totals_index = data.index(u'Total T\xe9rmico')
138     find_totals_end = data.index(u'Total Programado')
139 
140     ufthermal = data[find_thermal_index+3:find_totals_index-59]
141     total_data = data[find_totals_index:find_totals_end]
142 
143     #Remove all comapany names.
144     for val in ufthermal:
145         if ':' in val:
146             i = ufthermal.index(val)
147             del ufthermal[i:i+3]
148     
149     formatted_thermal = chunker([floater(item) for item in ufthermal])
150     mapped_totals = [total_mapping.get(x,x) for x in total_data]
151     formatted_totals = chunker([floater(item) for item in mapped_totals])
152 
153     return {'totals': formatted_totals, 'thermal': formatted_thermal}
154 
155 
156 def data_parser(formatted_data):
157     """
158     Converts formatted data into a pandas dataframe.  Removes any empty rows.
159     Returns a DataFrame.
160     """
161 
162     hours = range(1,24) + [0] + [25, 26]
163     dft = pd.DataFrame(formatted_data, index = hours)
164 
165     dft = dft.drop(dft.index[[-1,-2]])
166     dft = dft.replace(u'', np.nan)
167     dft = dft.dropna(how = 'all')
168 
169     return dft
170 
171 
172 def thermal_production(df):
173     """
174     Takes DataFrame and finds thermal generation for each hour.
175     Removes any non generating plants then maps plants to type.
176     Sums type instances and returns a dictionary.    
177     """
178 
179     therms = []
180     for hour in df.index.values:
181         dt=hour
182         currentt = df.loc[[hour]]
183 
184         #Create current plant output.
185         tp = {}
186         for item in list(df):
187             v = currentt.iloc[0][item]
188             tp[item] = v
189 
190         current_plants = {k: tp[k] for k in tp if not isnan(tp[k])}
191         mapped_plants = [(thermal_plants.get(plant, 'unknown'), val) for plant, val in current_plants.iteritems()]
192 
193         thermalDict = defaultdict(lambda: 0.0)
194 
195         #Sum values for duplicate keys.
196         for key,val in mapped_plants:
197             thermalDict[key] += val
198 
199         thermalDict['datetime'] = dt
200         thermalDict = dict(thermalDict)
201         therms.append(thermalDict)
202     
203     return therms
204 
205 
206 def total_production(df):
207     """
208     Takes DataFrame and finds generation totals for each hour.
209     Returns a dictionary.
210     """
211 
212     vals = []
213     #The Dominican Republic does not observe daylight savings time.
214     for hour in df.index.values:
215         dt=hour
216         current = df.loc[[hour]]
217         hydro = current.iloc[0]['Hydro']
218         wind = current.iloc[0]['Wind']
219 
220         #Wind and hydro totals do not always update exactly on the new hour.
221         #In this case we set them to None because they are unknown rather than zero.
222         if isnan(wind):
223             wind = None
224         if isnan(hydro):
225             hydro = None
226         
227         prod = {'wind': wind, 'hydro': hydro, 'datetime': dt}
228         vals.append(prod)
229 
230     return vals
231 
232 
233 def merge_production(thermal, total):
234     """
235     Takes thermal generation and total generation and merges them using 'datetime' key.
236     Returns a defaultdict.
237     """
238 
239     d = defaultdict(dict)
240     for each in (thermal, total):
241         for elem in each:
242             d[elem['datetime']].update(elem)
243 
244     final = sorted(d.values(), key=itemgetter("datetime"))
245 
246     def get_datetime(hour):
247         at = arrow.now('UTC-4')
248         dt = (at.replace(hour = int(hour), minute = 0, second = 0)).datetime
249         return dt
250     
251     for item in final:
252         i = item['datetime']
253         j = get_datetime(i)
254         item['datetime'] = j
255 
256     return final
257 
258 
259 def fetch_production(country_code = 'DO', session = None):
260     """
261     Requests the last known production mix (in MW) of a given country
262     Arguments:
263     country_code (optional) -- used in case a parser is able to fetch multiple countries
264     Return:
265     A dictionary in the form:
266     {
267       'countryCode': 'FR',
268       'datetime': '2017-01-01T00:00:00Z',
269       'production': {
270           'biomass': 0.0,
271           'coal': 0.0,
272           'gas': 0.0,
273           'hydro': 0.0,
274           'nuclear': null,
275           'oil': 0.0,
276           'solar': 0.0,
277           'wind': 0.0,
278           'geothermal': 0.0,
279           'unknown': 0.0
280       },
281       'storage': {
282           'hydro': -10.0,
283       },
284       'source': 'mysource.com'
285     }
286     """
287 
288     dat = data_formatter(get_data(session = None))
289     tot = data_parser(dat['totals'])
290     th = data_parser(dat['thermal'])
291     thermal = thermal_production(th)
292     total = total_production(tot)
293     merge = merge_production(thermal, total)
294 
295     production_mix_by_hour = []
296     for hour in merge:
297         production_mix = {
298           'countryCode': country_code,
299           'datetime': hour['datetime'],
300           'production': {
301               'biomass': hour.get('biomass', 0.0),
302               'coal': hour.get('coal', 0.0),
303               'gas': hour.get('gas', 0.0),
304               'hydro': hour['hydro'],
305               'nuclear': 0.0,
306               'oil': hour.get('oil', 0.0),
307               'solar': None,
308               'wind': hour['wind'],
309               'geothermal': 0.0,
310               'unknown': hour.get('unknown', 0.0)
311           },
312           'storage': {
313               'hydro': None,
314           },
315           'source': 'oc.org.do'
316         }
317         production_mix_by_hour.append(production_mix)
318         
319     return production_mix_by_hour
320     
321 
322 if __name__ ==  '__main__':
323     """Main method, never used by the Electricity Map backend, but handy for testing."""
324 
325     print('fetch_production() ->')
326     print(fetch_production())
327
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/parsers/DO.py b/parsers/DO.py
--- a/parsers/DO.py
+++ b/parsers/DO.py
@@ -140,7 +140,7 @@
     ufthermal = data[find_thermal_index+3:find_totals_index-59]
     total_data = data[find_totals_index:find_totals_end]
 
-    #Remove all comapany names.
+    #Remove all company names.
     for val in ufthermal:
         if ':' in val:
             i = ufthermal.index(val)
@@ -301,11 +301,11 @@
               'biomass': hour.get('biomass', 0.0),
               'coal': hour.get('coal', 0.0),
               'gas': hour.get('gas', 0.0),
-              'hydro': hour['hydro'],
+              'hydro': hour.get('hydro', 0.0),
               'nuclear': 0.0,
               'oil': hour.get('oil', 0.0),
               'solar': None,
-              'wind': hour['wind'],
+              'wind': hour.get('wind', 0.0),
               'geothermal': 0.0,
               'unknown': hour.get('unknown', 0.0)
           },

{"golden_diff": "diff --git a/parsers/DO.py b/parsers/DO.py\n--- a/parsers/DO.py\n+++ b/parsers/DO.py\n@@ -140,7 +140,7 @@\n     ufthermal = data[find_thermal_index+3:find_totals_index-59]\n     total_data = data[find_totals_index:find_totals_end]\n \n-    #Remove all comapany names.\n+    #Remove all company names.\n     for val in ufthermal:\n         if ':' in val:\n             i = ufthermal.index(val)\n@@ -301,11 +301,11 @@\n               'biomass': hour.get('biomass', 0.0),\n               'coal': hour.get('coal', 0.0),\n               'gas': hour.get('gas', 0.0),\n-              'hydro': hour['hydro'],\n+              'hydro': hour.get('hydro', 0.0),\n               'nuclear': 0.0,\n               'oil': hour.get('oil', 0.0),\n               'solar': None,\n-              'wind': hour['wind'],\n+              'wind': hour.get('wind', 0.0),\n               'geothermal': 0.0,\n               'unknown': hour.get('unknown', 0.0)\n           },\n", "issue": "Hydro missing from DO\n![opbeat com-tomorrow-electricity-map-backend-errors-329-](https://user-images.githubusercontent.com/1655848/29333158-11e0dec0-8203-11e7-9e38-a24c2d74e42f.png)\r\n\n", "before_files": [{"content": "#!/usr/bin/env python\n\nimport arrow\nfrom bs4 import BeautifulSoup\nfrom collections import defaultdict\nimport itertools\nfrom math import isnan\nimport numpy as np\nfrom operator import itemgetter\nimport pandas as pd\nimport requests\n\n\n#This parser gets hourly electricity generation data from oc.org.do for the Dominican Republic.\n#The data is in MWh but since it is updated hourly we can view it as MW.\n#Solar generation has no data available currently but multiple projects are planned/under construction.\n\nurl = 'http://184.168.74.190:81/ReportesGraficos/ReportePostdespacho.aspx'\n\ntotal_mapping = {\n                u'Total T\\xe9rmico': 'Thermal',\n                u'Total E\\xf3lico': 'Wind',\n                u'Total Hidroel\\xe9ctrica': 'Hydro',\n                'Total Generado': 'Generated'\n                }\n\n#Power plant types\n#http://www.sie.gob.do/images/Estadisticas/MEM/GeneracionDiariaEnero2017/\n#Reporte_diario_de_generacion_31_enero_2017_merged2.pdf\n\nthermal_plants = {\n                 u'AES ANDRES': 'gas',\n                 u'BARAHONA CARBON': 'coal',\n                 u'BERSAL': 'unknown',\n                 u'CEPP 1': 'oil',\n                 u'CEPP 2': 'oil',\n                 u'CESPM 1': 'oil',\n                 u'CESPM 2': 'oil',\n                 u'CESPM 3': 'oil',\n                 u'ESTRELLA DEL MAR 2 CFO': 'oil',\n                 u'ESTRELLA DEL MAR 2 CGN': 'gas',\n                 u'ESTRELLA DEL MAR 2 SFO': 'oil',\n                 u'ESTRELLA DEL MAR 2 SGN': 'gas',\n                 u'HAINA TG': 'oil',\n                 u'INCA KM22': 'oil',\n                 u'ITABO 1': 'coal',\n                 u'ITABO 2': 'coal',\n                 u'LA VEGA': 'oil',\n                 u'LOS MINA 5': 'gas',\n                 u'LOS MINA 6': 'gas',\n                 u'LOS MINA 7': 'gas',\n                 u'LOS OR\\xcdGENES POWER PLANT FUEL OIL': 'oil',\n                 u'LOS OR\\xcdGENES POWER PLANT GAS NATURAL': 'gas',\n                 u'METALDOM': 'oil',\n                 u'MONTE PLATA SOLAR': 'solar',\n                 u'MONTE RIO': 'oil',\n                 u'PALAMARA': 'oil',\n                 u'PARQUE ENERGETICO LOS MINA CC PARCIAL': 'gas',\n                 u'PARQUE ENERGETICO LOS MINA CC TOTAL': 'gas',\n                 u'PIMENTEL 1': 'oil',\n                 u'PIMENTEL 2': 'oil',\n                 u'PIMENTEL 3': 'oil',\n                 u'QUISQUEYA 1': 'gas',\n                 u'QUISQUEYA 2': 'gas',\n                 u'RIO SAN JUAN': 'oil',\n                 u'SAN FELIPE': 'oil',\n                 u'SAN FELIPE CC': 'gas',\n                 u'SAN FELIPE VAP': 'oil',\n                 u'SAN LORENZO 1': 'gas',\n                 u'SAN PEDRO BIO-ENERGY': 'biomass',\n                 u'SAN PEDRO VAPOR': 'oil',\n                 u'SULTANA DEL ESTE': 'oil'\n                 }\n\n\ndef get_data(session = None):\n    \"\"\"\n    Makes a request to source url.\n    Finds main table and creates a list of all table elements in unicode string format.\n    Returns a list.\n    \"\"\"\n\n    data = []\n    s = session or requests.Session()\n    data_req = s.get(url)\n    soup = BeautifulSoup(data_req.content, 'lxml')\n\n    tbs = soup.find(\"table\", id=\"PostdespachoUnidadesTermicasGrid_DXMainTable\")\n    rows = tbs.find_all(\"td\")\n\n    for row in rows:\n        num = row.getText().strip()\n        data.append(unicode(num))\n\n    return data\n\n\ndef floater(item):\n    \"\"\"\n    Attempts to convert any item given to a float.  Returns item if it fails.\n    \"\"\"\n\n    try:\n        return float(item)\n    except ValueError:\n        return item\n\n\ndef chunker(big_lst):\n    \"\"\"\n    Breaks a big list into a list of lists.  Removes any list with no data then turns remaining\n    lists into key: value pairs with first element from the list being the key.\n    Returns a dictionary.\n    \"\"\"\n    \n    chunks = [big_lst[x:x+27] for x in xrange(0, len(big_lst), 27)]\n\n    #Remove the list if it contains no data.\n    for chunk in chunks:\n        if any(chunk) == True:\n            continue\n        else:\n            chunks.remove(chunk)\n\n    chunked_list = {words[0]:words[1:] for words in chunks}\n\n    return chunked_list\n\n\ndef data_formatter(data):\n    \"\"\"\n    Takes data and finds relevant sections.  Formats and breaks data into usable parts.\n    Returns a nested dictionary.\n    \"\"\"\n\n    find_thermal_index = data.index(u'GRUPO: T\\xe9rmica')\n    find_totals_index = data.index(u'Total T\\xe9rmico')\n    find_totals_end = data.index(u'Total Programado')\n\n    ufthermal = data[find_thermal_index+3:find_totals_index-59]\n    total_data = data[find_totals_index:find_totals_end]\n\n    #Remove all comapany names.\n    for val in ufthermal:\n        if ':' in val:\n            i = ufthermal.index(val)\n            del ufthermal[i:i+3]\n    \n    formatted_thermal = chunker([floater(item) for item in ufthermal])\n    mapped_totals = [total_mapping.get(x,x) for x in total_data]\n    formatted_totals = chunker([floater(item) for item in mapped_totals])\n\n    return {'totals': formatted_totals, 'thermal': formatted_thermal}\n\n\ndef data_parser(formatted_data):\n    \"\"\"\n    Converts formatted data into a pandas dataframe.  Removes any empty rows.\n    Returns a DataFrame.\n    \"\"\"\n\n    hours = range(1,24) + [0] + [25, 26]\n    dft = pd.DataFrame(formatted_data, index = hours)\n\n    dft = dft.drop(dft.index[[-1,-2]])\n    dft = dft.replace(u'', np.nan)\n    dft = dft.dropna(how = 'all')\n\n    return dft\n\n\ndef thermal_production(df):\n    \"\"\"\n    Takes DataFrame and finds thermal generation for each hour.\n    Removes any non generating plants then maps plants to type.\n    Sums type instances and returns a dictionary.    \n    \"\"\"\n\n    therms = []\n    for hour in df.index.values:\n        dt=hour\n        currentt = df.loc[[hour]]\n\n        #Create current plant output.\n        tp = {}\n        for item in list(df):\n            v = currentt.iloc[0][item]\n            tp[item] = v\n\n        current_plants = {k: tp[k] for k in tp if not isnan(tp[k])}\n        mapped_plants = [(thermal_plants.get(plant, 'unknown'), val) for plant, val in current_plants.iteritems()]\n\n        thermalDict = defaultdict(lambda: 0.0)\n\n        #Sum values for duplicate keys.\n        for key,val in mapped_plants:\n            thermalDict[key] += val\n\n        thermalDict['datetime'] = dt\n        thermalDict = dict(thermalDict)\n        therms.append(thermalDict)\n    \n    return therms\n\n\ndef total_production(df):\n    \"\"\"\n    Takes DataFrame and finds generation totals for each hour.\n    Returns a dictionary.\n    \"\"\"\n\n    vals = []\n    #The Dominican Republic does not observe daylight savings time.\n    for hour in df.index.values:\n        dt=hour\n        current = df.loc[[hour]]\n        hydro = current.iloc[0]['Hydro']\n        wind = current.iloc[0]['Wind']\n\n        #Wind and hydro totals do not always update exactly on the new hour.\n        #In this case we set them to None because they are unknown rather than zero.\n        if isnan(wind):\n            wind = None\n        if isnan(hydro):\n            hydro = None\n        \n        prod = {'wind': wind, 'hydro': hydro, 'datetime': dt}\n        vals.append(prod)\n\n    return vals\n\n\ndef merge_production(thermal, total):\n    \"\"\"\n    Takes thermal generation and total generation and merges them using 'datetime' key.\n    Returns a defaultdict.\n    \"\"\"\n\n    d = defaultdict(dict)\n    for each in (thermal, total):\n        for elem in each:\n            d[elem['datetime']].update(elem)\n\n    final = sorted(d.values(), key=itemgetter(\"datetime\"))\n\n    def get_datetime(hour):\n        at = arrow.now('UTC-4')\n        dt = (at.replace(hour = int(hour), minute = 0, second = 0)).datetime\n        return dt\n    \n    for item in final:\n        i = item['datetime']\n        j = get_datetime(i)\n        item['datetime'] = j\n\n    return final\n\n\ndef fetch_production(country_code = 'DO', session = None):\n    \"\"\"\n    Requests the last known production mix (in MW) of a given country\n    Arguments:\n    country_code (optional) -- used in case a parser is able to fetch multiple countries\n    Return:\n    A dictionary in the form:\n    {\n      'countryCode': 'FR',\n      'datetime': '2017-01-01T00:00:00Z',\n      'production': {\n          'biomass': 0.0,\n          'coal': 0.0,\n          'gas': 0.0,\n          'hydro': 0.0,\n          'nuclear': null,\n          'oil': 0.0,\n          'solar': 0.0,\n          'wind': 0.0,\n          'geothermal': 0.0,\n          'unknown': 0.0\n      },\n      'storage': {\n          'hydro': -10.0,\n      },\n      'source': 'mysource.com'\n    }\n    \"\"\"\n\n    dat = data_formatter(get_data(session = None))\n    tot = data_parser(dat['totals'])\n    th = data_parser(dat['thermal'])\n    thermal = thermal_production(th)\n    total = total_production(tot)\n    merge = merge_production(thermal, total)\n\n    production_mix_by_hour = []\n    for hour in merge:\n        production_mix = {\n          'countryCode': country_code,\n          'datetime': hour['datetime'],\n          'production': {\n              'biomass': hour.get('biomass', 0.0),\n              'coal': hour.get('coal', 0.0),\n              'gas': hour.get('gas', 0.0),\n              'hydro': hour['hydro'],\n              'nuclear': 0.0,\n              'oil': hour.get('oil', 0.0),\n              'solar': None,\n              'wind': hour['wind'],\n              'geothermal': 0.0,\n              'unknown': hour.get('unknown', 0.0)\n          },\n          'storage': {\n              'hydro': None,\n          },\n          'source': 'oc.org.do'\n        }\n        production_mix_by_hour.append(production_mix)\n        \n    return production_mix_by_hour\n    \n\nif __name__ ==  '__main__':\n    \"\"\"Main method, never used by the Electricity Map backend, but handy for testing.\"\"\"\n\n    print('fetch_production() ->')\n    print(fetch_production())\n", "path": "parsers/DO.py"}], "after_files": [{"content": "#!/usr/bin/env python\n\nimport arrow\nfrom bs4 import BeautifulSoup\nfrom collections import defaultdict\nimport itertools\nfrom math import isnan\nimport numpy as np\nfrom operator import itemgetter\nimport pandas as pd\nimport requests\n\n\n#This parser gets hourly electricity generation data from oc.org.do for the Dominican Republic.\n#The data is in MWh but since it is updated hourly we can view it as MW.\n#Solar generation has no data available currently but multiple projects are planned/under construction.\n\nurl = 'http://184.168.74.190:81/ReportesGraficos/ReportePostdespacho.aspx'\n\ntotal_mapping = {\n                u'Total T\\xe9rmico': 'Thermal',\n                u'Total E\\xf3lico': 'Wind',\n                u'Total Hidroel\\xe9ctrica': 'Hydro',\n                'Total Generado': 'Generated'\n                }\n\n#Power plant types\n#http://www.sie.gob.do/images/Estadisticas/MEM/GeneracionDiariaEnero2017/\n#Reporte_diario_de_generacion_31_enero_2017_merged2.pdf\n\nthermal_plants = {\n                 u'AES ANDRES': 'gas',\n                 u'BARAHONA CARBON': 'coal',\n                 u'BERSAL': 'unknown',\n                 u'CEPP 1': 'oil',\n                 u'CEPP 2': 'oil',\n                 u'CESPM 1': 'oil',\n                 u'CESPM 2': 'oil',\n                 u'CESPM 3': 'oil',\n                 u'ESTRELLA DEL MAR 2 CFO': 'oil',\n                 u'ESTRELLA DEL MAR 2 CGN': 'gas',\n                 u'ESTRELLA DEL MAR 2 SFO': 'oil',\n                 u'ESTRELLA DEL MAR 2 SGN': 'gas',\n                 u'HAINA TG': 'oil',\n                 u'INCA KM22': 'oil',\n                 u'ITABO 1': 'coal',\n                 u'ITABO 2': 'coal',\n                 u'LA VEGA': 'oil',\n                 u'LOS MINA 5': 'gas',\n                 u'LOS MINA 6': 'gas',\n                 u'LOS MINA 7': 'gas',\n                 u'LOS OR\\xcdGENES POWER PLANT FUEL OIL': 'oil',\n                 u'LOS OR\\xcdGENES POWER PLANT GAS NATURAL': 'gas',\n                 u'METALDOM': 'oil',\n                 u'MONTE PLATA SOLAR': 'solar',\n                 u'MONTE RIO': 'oil',\n                 u'PALAMARA': 'oil',\n                 u'PARQUE ENERGETICO LOS MINA CC PARCIAL': 'gas',\n                 u'PARQUE ENERGETICO LOS MINA CC TOTAL': 'gas',\n                 u'PIMENTEL 1': 'oil',\n                 u'PIMENTEL 2': 'oil',\n                 u'PIMENTEL 3': 'oil',\n                 u'QUISQUEYA 1': 'gas',\n                 u'QUISQUEYA 2': 'gas',\n                 u'RIO SAN JUAN': 'oil',\n                 u'SAN FELIPE': 'oil',\n                 u'SAN FELIPE CC': 'gas',\n                 u'SAN FELIPE VAP': 'oil',\n                 u'SAN LORENZO 1': 'gas',\n                 u'SAN PEDRO BIO-ENERGY': 'biomass',\n                 u'SAN PEDRO VAPOR': 'oil',\n                 u'SULTANA DEL ESTE': 'oil'\n                 }\n\n\ndef get_data(session = None):\n    \"\"\"\n    Makes a request to source url.\n    Finds main table and creates a list of all table elements in unicode string format.\n    Returns a list.\n    \"\"\"\n\n    data = []\n    s = session or requests.Session()\n    data_req = s.get(url)\n    soup = BeautifulSoup(data_req.content, 'lxml')\n\n    tbs = soup.find(\"table\", id=\"PostdespachoUnidadesTermicasGrid_DXMainTable\")\n    rows = tbs.find_all(\"td\")\n\n    for row in rows:\n        num = row.getText().strip()\n        data.append(unicode(num))\n\n    return data\n\n\ndef floater(item):\n    \"\"\"\n    Attempts to convert any item given to a float.  Returns item if it fails.\n    \"\"\"\n\n    try:\n        return float(item)\n    except ValueError:\n        return item\n\n\ndef chunker(big_lst):\n    \"\"\"\n    Breaks a big list into a list of lists.  Removes any list with no data then turns remaining\n    lists into key: value pairs with first element from the list being the key.\n    Returns a dictionary.\n    \"\"\"\n    \n    chunks = [big_lst[x:x+27] for x in xrange(0, len(big_lst), 27)]\n\n    #Remove the list if it contains no data.\n    for chunk in chunks:\n        if any(chunk) == True:\n            continue\n        else:\n            chunks.remove(chunk)\n\n    chunked_list = {words[0]:words[1:] for words in chunks}\n\n    return chunked_list\n\n\ndef data_formatter(data):\n    \"\"\"\n    Takes data and finds relevant sections.  Formats and breaks data into usable parts.\n    Returns a nested dictionary.\n    \"\"\"\n\n    find_thermal_index = data.index(u'GRUPO: T\\xe9rmica')\n    find_totals_index = data.index(u'Total T\\xe9rmico')\n    find_totals_end = data.index(u'Total Programado')\n\n    ufthermal = data[find_thermal_index+3:find_totals_index-59]\n    total_data = data[find_totals_index:find_totals_end]\n\n    #Remove all company names.\n    for val in ufthermal:\n        if ':' in val:\n            i = ufthermal.index(val)\n            del ufthermal[i:i+3]\n    \n    formatted_thermal = chunker([floater(item) for item in ufthermal])\n    mapped_totals = [total_mapping.get(x,x) for x in total_data]\n    formatted_totals = chunker([floater(item) for item in mapped_totals])\n\n    return {'totals': formatted_totals, 'thermal': formatted_thermal}\n\n\ndef data_parser(formatted_data):\n    \"\"\"\n    Converts formatted data into a pandas dataframe.  Removes any empty rows.\n    Returns a DataFrame.\n    \"\"\"\n\n    hours = range(1,24) + [0] + [25, 26]\n    dft = pd.DataFrame(formatted_data, index = hours)\n\n    dft = dft.drop(dft.index[[-1,-2]])\n    dft = dft.replace(u'', np.nan)\n    dft = dft.dropna(how = 'all')\n\n    return dft\n\n\ndef thermal_production(df):\n    \"\"\"\n    Takes DataFrame and finds thermal generation for each hour.\n    Removes any non generating plants then maps plants to type.\n    Sums type instances and returns a dictionary.    \n    \"\"\"\n\n    therms = []\n    for hour in df.index.values:\n        dt=hour\n        currentt = df.loc[[hour]]\n\n        #Create current plant output.\n        tp = {}\n        for item in list(df):\n            v = currentt.iloc[0][item]\n            tp[item] = v\n\n        current_plants = {k: tp[k] for k in tp if not isnan(tp[k])}\n        mapped_plants = [(thermal_plants.get(plant, 'unknown'), val) for plant, val in current_plants.iteritems()]\n\n        thermalDict = defaultdict(lambda: 0.0)\n\n        #Sum values for duplicate keys.\n        for key,val in mapped_plants:\n            thermalDict[key] += val\n\n        thermalDict['datetime'] = dt\n        thermalDict = dict(thermalDict)\n        therms.append(thermalDict)\n    \n    return therms\n\n\ndef total_production(df):\n    \"\"\"\n    Takes DataFrame and finds generation totals for each hour.\n    Returns a dictionary.\n    \"\"\"\n\n    vals = []\n    #The Dominican Republic does not observe daylight savings time.\n    for hour in df.index.values:\n        dt=hour\n        current = df.loc[[hour]]\n        hydro = current.iloc[0]['Hydro']\n        wind = current.iloc[0]['Wind']\n\n        #Wind and hydro totals do not always update exactly on the new hour.\n        #In this case we set them to None because they are unknown rather than zero.\n        if isnan(wind):\n            wind = None\n        if isnan(hydro):\n            hydro = None\n        \n        prod = {'wind': wind, 'hydro': hydro, 'datetime': dt}\n        vals.append(prod)\n\n    return vals\n\n\ndef merge_production(thermal, total):\n    \"\"\"\n    Takes thermal generation and total generation and merges them using 'datetime' key.\n    Returns a defaultdict.\n    \"\"\"\n\n    d = defaultdict(dict)\n    for each in (thermal, total):\n        for elem in each:\n            d[elem['datetime']].update(elem)\n\n    final = sorted(d.values(), key=itemgetter(\"datetime\"))\n\n    def get_datetime(hour):\n        at = arrow.now('UTC-4')\n        dt = (at.replace(hour = int(hour), minute = 0, second = 0)).datetime\n        return dt\n    \n    for item in final:\n        i = item['datetime']\n        j = get_datetime(i)\n        item['datetime'] = j\n\n    return final\n\n\ndef fetch_production(country_code = 'DO', session = None):\n    \"\"\"\n    Requests the last known production mix (in MW) of a given country\n    Arguments:\n    country_code (optional) -- used in case a parser is able to fetch multiple countries\n    Return:\n    A dictionary in the form:\n    {\n      'countryCode': 'FR',\n      'datetime': '2017-01-01T00:00:00Z',\n      'production': {\n          'biomass': 0.0,\n          'coal': 0.0,\n          'gas': 0.0,\n          'hydro': 0.0,\n          'nuclear': null,\n          'oil': 0.0,\n          'solar': 0.0,\n          'wind': 0.0,\n          'geothermal': 0.0,\n          'unknown': 0.0\n      },\n      'storage': {\n          'hydro': -10.0,\n      },\n      'source': 'mysource.com'\n    }\n    \"\"\"\n\n    dat = data_formatter(get_data(session = None))\n    tot = data_parser(dat['totals'])\n    th = data_parser(dat['thermal'])\n    thermal = thermal_production(th)\n    total = total_production(tot)\n    merge = merge_production(thermal, total)\n\n    production_mix_by_hour = []\n    for hour in merge:\n        production_mix = {\n          'countryCode': country_code,\n          'datetime': hour['datetime'],\n          'production': {\n              'biomass': hour.get('biomass', 0.0),\n              'coal': hour.get('coal', 0.0),\n              'gas': hour.get('gas', 0.0),\n              'hydro': hour.get('hydro', 0.0),\n              'nuclear': 0.0,\n              'oil': hour.get('oil', 0.0),\n              'solar': None,\n              'wind': hour.get('wind', 0.0),\n              'geothermal': 0.0,\n              'unknown': hour.get('unknown', 0.0)\n          },\n          'storage': {\n              'hydro': None,\n          },\n          'source': 'oc.org.do'\n        }\n        production_mix_by_hour.append(production_mix)\n        \n    return production_mix_by_hour\n    \n\nif __name__ ==  '__main__':\n    \"\"\"Main method, never used by the Electricity Map backend, but handy for testing.\"\"\"\n\n    print('fetch_production() ->')\n    print(fetch_production())\n", "path": "parsers/DO.py"}]}