We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
California is down due to negative nuclear production
[Kibana](https://kibana.electricitymap.org/app/kibana#/discover/10af54f0-0c4a-11e9-85c1-1d63df8c862c?_g=(refreshInterval:('$$hashKey':'object:232',display:'5%20minutes',pause:!f,section:2,value:300000),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(message,extra.key,level),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'96f67170-0c49-11e9-85c1-1d63df8c862c',key:level,negate:!f,params:(query:ERROR,type:phrase),type:phrase,value:ERROR),query:(match:(level:(query:ERROR,type:phrase)))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'96f67170-0c49-11e9-85c1-1d63df8c862c',key:extra.key,negate:!f,params:(query:US-CAL-CISO,type:phrase),type:phrase,value:US-CAL-CISO),query:(match:(extra.key:(query:US-CAL-CISO,type:phrase))))),index:'96f67170-0c49-11e9-85c1-1d63df8c862c',interval:auto,query:(language:lucene,query:''),sort:!('@timestamp',desc))):
>invalid point: {'zoneKey': 'US-CAL-CISO', 'production': defaultdict(<class 'float'>, {'solar': 0.0, 'wind': 35.0, 'geothermal': 537.0, 'biomass': 385.0, 'hydro': 916.0, 'coal': 16.0, 'nuclear': -34.0, 'gas': 10688.0, 'unknown': 0.0}), 'storage': defaultdict(<class 'float'>, {'battery': 3.0}), 'source': 'caiso.com', 'datetime': datetime.datetime(2020, 10, 28, 4, 10, tzinfo=tzfile('/usr/share/zoneinfo/US/Pacific')), 'schemaVersion': 3}, reason:US-CAL-CISO: key nuclear has negative value -34.0

Given the small value and that it's zero on the [CAISO web site](http://www.caiso.com/TodaysOutlook/Pages/supply.aspx), this is probably self consumption. Some googling shows that both units of Diablo Canyon, California's sole nuclear power plant, are [down for maintainance](https://keyt.com/news/san-luis-obispo-county/2020/10/15/pge-removes-diablo-canyons-unit-2-for-unexpected-maintenance/).

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `parsers/US_CA.py`
Content:
```
1 #!/usr/bin/env python3
2 
3 import arrow
4 import pandas
5 import requests
6 from bs4 import BeautifulSoup
7 from collections import defaultdict
8 
9 FUEL_SOURCE_CSV = 'http://www.caiso.com/outlook/SP/fuelsource.csv'
10 
11 MX_EXCHANGE_URL = 'http://www.cenace.gob.mx/Paginas/Publicas/Info/DemandaRegional.aspx'
12 
13 def fetch_production(zone_key='US-CA', session=None, target_datetime=None,
14                      logger=None):
15     """Requests the last known production mix (in MW) of a given country
16 
17     Arguments:
18     zone_key: used in case a parser is able to fetch multiple countries
19     session: request session passed in order to re-use an existing session
20 
21     Return:
22     A dictionary in the form:
23     {
24       'zoneKey': 'FR',
25       'datetime': '2017-01-01T00:00:00Z',
26       'production': {
27           'biomass': 0.0,
28           'coal': 0.0,
29           'gas': 0.0,
30           'hydro': 0.0,
31           'nuclear': null,
32           'oil': 0.0,
33           'solar': 0.0,
34           'wind': 0.0,
35           'geothermal': 0.0,
36           'unknown': 0.0
37       },
38       'storage': {
39           'hydro': -10.0,
40       },
41       'source': 'mysource.com'
42     }
43     """
44     if target_datetime:
45         return fetch_historical_production(target_datetime, zone_key)
46 
47     target_datetime = arrow.get(target_datetime)
48 
49     # Get the production from the CSV
50     csv = pandas.read_csv(FUEL_SOURCE_CSV)
51     latest_index = len(csv) - 1
52     production_map = {
53         'Solar': 'solar',
54         'Wind': 'wind',
55         'Geothermal': 'geothermal',
56         'Biomass': 'biomass',
57         'Biogas': 'biomass',
58         'Small hydro': 'hydro',
59         'Coal': 'coal',
60         'Nuclear': 'nuclear',
61         'Natural gas': 'gas',
62         'Large hydro': 'hydro',
63         'Other': 'unknown'
64     }
65     storage_map = {
66         'Batteries': 'battery'
67     }
68     daily_data = []
69     for i in range(0, latest_index + 1):
70         h, m = map(int, csv['Time'][i].split(':'))
71         date = arrow.utcnow().to('US/Pacific').replace(hour=h, minute=m,
72                                                        second=0, microsecond=0)
73         data = {
74             'zoneKey': zone_key,
75             'production': defaultdict(float),
76             'storage': defaultdict(float),
77             'source': 'caiso.com',
78             'datetime': date.datetime
79         }
80 
81         # map items from names in CAISO CSV to names used in Electricity Map
82         for ca_gen_type, mapped_gen_type in production_map.items():
83             production = float(csv[ca_gen_type][i])
84             
85             if mapped_gen_type == 'solar' and production < 0:
86                 logger.warn('Solar production for US_CA was reported as less than 0 and was clamped')
87                 production = 0.0
88             
89             # if another mean of production created a value, sum them up
90             data['production'][mapped_gen_type] += production
91 
92         for ca_storage_type, mapped_storage_type in storage_map.items():
93             storage = -float(csv[ca_storage_type][i])
94 
95             # if another mean of storage created a value, sum them up
96             data['storage'][mapped_storage_type] += storage
97 
98         daily_data.append(data)
99 
100     return daily_data
101 
102 
103 def fetch_historical_production(target_datetime, zone_key):
104     return fetch_historical_data(target_datetime, zone_key)[0]
105 
106 
107 def fetch_historical_exchange(target_datetime):
108     return fetch_historical_data(target_datetime)[1]
109 
110 
111 def fetch_historical_data(target_datetime, zone_key='US-CA'):
112     # caiso.com provides daily data until the day before today
113     # get a clean date at the beginning of yesterday
114     target_date = arrow.get(target_datetime).to('US/Pacific').replace(
115         hour=0, minute=0, second=0, microsecond=0)
116 
117     url = 'http://content.caiso.com/green/renewrpt/' + target_date.format(
118         'YYYYMMDD') + '_DailyRenewablesWatch.txt'
119 
120     renewable_resources = pandas.read_table(
121         url, sep='\t\t', skiprows=2, header=None,
122         names=['Hour', 'GEOTHERMAL', 'BIOMASS', 'BIOGAS', 'SMALL HYDRO',
123                'WIND TOTAL', 'SOLAR PV', 'SOLAR THERMAL'],
124         skipfooter=27, skipinitialspace=True, engine='python')
125     other_resources = pandas.read_table(
126         url, sep='\t\t', skiprows=30, header=None,
127         names=['Hour', 'RENEWABLES', 'NUCLEAR', 'THERMAL', 'IMPORTS', 'HYDRO'],
128         skipinitialspace=True, engine='python')
129 
130     daily_data, import_data = [], []
131 
132     for i in range(0, 24):
133         daily_data.append({
134             'zoneKey': zone_key,
135             'storage': {},
136             'source': 'caiso.com',
137             'production': {
138                 'biomass': float(renewable_resources['BIOMASS'][i]),
139                 'gas': float(renewable_resources['BIOGAS'][i])
140                        + float(other_resources['THERMAL'][i]),
141                 'hydro': float(renewable_resources['SMALL HYDRO'][i])
142                          + float(other_resources['HYDRO'][i]),
143                 'nuclear': float(other_resources['NUCLEAR'][i]),
144                 'solar': float(renewable_resources['SOLAR PV'][i])
145                          + float(renewable_resources['SOLAR THERMAL'][i]),
146                 'wind': float(renewable_resources['WIND TOTAL'][i]),
147                 'geothermal': float(renewable_resources['GEOTHERMAL'][i]),
148             },
149             'datetime': target_date.shift(hours=i + 1).datetime,
150         })
151         import_data.append(
152             {
153                 'sortedZoneKeys': 'US->US-CA',
154                 'datetime': target_date.shift(hours=i + 1).datetime,
155                 'netFlow': float(other_resources['IMPORTS'][i]),
156                 'source': 'caiso.com'
157             }
158         )
159 
160     return daily_data, import_data
161 
162 
163 def fetch_MX_exchange(s):
164     req = s.get(MX_EXCHANGE_URL)
165     soup = BeautifulSoup(req.text, 'html.parser')
166     exchange_div = soup.find("div", attrs={'id': 'IntercambioUSA-BCA'})
167     val = exchange_div.text
168 
169     # cenace html uses unicode hyphens instead of minus signs
170     try:
171         val = val.replace(chr(8208), chr(45))
172     except ValueError:
173         pass
174 
175     # negative value indicates flow from CA to MX
176 
177     return float(val)
178 
179 
180 def fetch_exchange(zone_key1, zone_key2, session=None, target_datetime=None,
181                    logger=None):
182     """Requests the last known power exchange (in MW) between two zones
183     Arguments:
184     zone_key1: the first country code
185     zone_key2: the second country code; order of the two codes in params
186       doesn't matter
187     session: request session passed in order to re-use an existing session
188     Return:
189     A dictionary in the form:
190     {
191       'sortedZoneKeys': 'DK->NO',
192       'datetime': '2017-01-01T00:00:00Z',
193       'netFlow': 0.0,
194       'source': 'mysource.com'
195     }
196     where net flow is from DK into NO
197     """
198     sorted_zone_keys = '->'.join(sorted([zone_key1, zone_key2]))
199 
200     s = session or requests.Session()
201 
202     if sorted_zone_keys == 'MX-BC->US-CA' or sorted_zone_keys == 'MX-BC->US-CAL-CISO':
203         netflow = fetch_MX_exchange(s)
204         exchange = {
205           'sortedZoneKeys': sorted_zone_keys,
206           'datetime': arrow.now('America/Tijuana').datetime,
207           'netFlow': netflow,
208           'source': 'cenace.gob.mx'
209         }
210         return exchange
211 
212     if target_datetime:
213         return fetch_historical_exchange(target_datetime)
214 
215     # CSV has imports to California as positive.
216     # Electricity Map expects A->B to indicate flow to B as positive.
217     # So values in CSV can be used as-is.
218 
219     csv = pandas.read_csv(FUEL_SOURCE_CSV)
220     latest_index = len(csv) - 1
221     daily_data = []
222     for i in range(0, latest_index + 1):
223         h, m = map(int, csv['Time'][i].split(':'))
224         date = arrow.utcnow().to('US/Pacific').replace(hour=h, minute=m,
225                                                        second=0, microsecond=0)
226         data = {
227             'sortedZoneKeys': sorted_zone_keys,
228             'datetime': date.datetime,
229             'netFlow': float(csv['Imports'][i]),
230             'source': 'caiso.com'
231         }
232 
233         daily_data.append(data)
234 
235     return daily_data
236 
237 
238 if __name__ == '__main__':
239     "Main method, not used by Electricity Map backend, but handy for testing"
240 
241     from pprint import pprint
242 
243     print('fetch_production() ->')
244     pprint(fetch_production())
245 
246     print('fetch_exchange("US-CA", "US") ->')
247     #pprint(fetch_exchange("US-CA", "US"))
248 
249     print('fetch_exchange("MX-BC", "US-CA")')
250     pprint(fetch_exchange("MX-BC", "US-CA"))
251
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/parsers/US_CA.py b/parsers/US_CA.py
--- a/parsers/US_CA.py
+++ b/parsers/US_CA.py
@@ -5,13 +5,14 @@
 import requests
 from bs4 import BeautifulSoup
 from collections import defaultdict
+import logging
 
 FUEL_SOURCE_CSV = 'http://www.caiso.com/outlook/SP/fuelsource.csv'
 
 MX_EXCHANGE_URL = 'http://www.cenace.gob.mx/Paginas/Publicas/Info/DemandaRegional.aspx'
 
 def fetch_production(zone_key='US-CA', session=None, target_datetime=None,
-                     logger=None):
+                     logger: logging.Logger = logging.getLogger(__name__)):
     """Requests the last known production mix (in MW) of a given country
 
     Arguments:
@@ -82,8 +83,8 @@
         for ca_gen_type, mapped_gen_type in production_map.items():
             production = float(csv[ca_gen_type][i])
             
-            if mapped_gen_type == 'solar' and production < 0:
-                logger.warn('Solar production for US_CA was reported as less than 0 and was clamped')
+            if production < 0 and (mapped_gen_type == 'solar' or mapped_gen_type == 'nuclear'):
+                logger.warn(ca_gen_type + ' production for US_CA was reported as less than 0 and was clamped')
                 production = 0.0
             
             # if another mean of production created a value, sum them up

{"golden_diff": "diff --git a/parsers/US_CA.py b/parsers/US_CA.py\n--- a/parsers/US_CA.py\n+++ b/parsers/US_CA.py\n@@ -5,13 +5,14 @@\n import requests\n from bs4 import BeautifulSoup\n from collections import defaultdict\n+import logging\n \n FUEL_SOURCE_CSV = 'http://www.caiso.com/outlook/SP/fuelsource.csv'\n \n MX_EXCHANGE_URL = 'http://www.cenace.gob.mx/Paginas/Publicas/Info/DemandaRegional.aspx'\n \n def fetch_production(zone_key='US-CA', session=None, target_datetime=None,\n-                     logger=None):\n+                     logger: logging.Logger = logging.getLogger(__name__)):\n     \"\"\"Requests the last known production mix (in MW) of a given country\n \n     Arguments:\n@@ -82,8 +83,8 @@\n         for ca_gen_type, mapped_gen_type in production_map.items():\n             production = float(csv[ca_gen_type][i])\n             \n-            if mapped_gen_type == 'solar' and production < 0:\n-                logger.warn('Solar production for US_CA was reported as less than 0 and was clamped')\n+            if production < 0 and (mapped_gen_type == 'solar' or mapped_gen_type == 'nuclear'):\n+                logger.warn(ca_gen_type + ' production for US_CA was reported as less than 0 and was clamped')\n                 production = 0.0\n             \n             # if another mean of production created a value, sum them up\n", "issue": "California is down due to negative nuclear production\n[Kibana](https://kibana.electricitymap.org/app/kibana#/discover/10af54f0-0c4a-11e9-85c1-1d63df8c862c?_g=(refreshInterval:('$$hashKey':'object:232',display:'5%20minutes',pause:!f,section:2,value:300000),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(message,extra.key,level),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'96f67170-0c49-11e9-85c1-1d63df8c862c',key:level,negate:!f,params:(query:ERROR,type:phrase),type:phrase,value:ERROR),query:(match:(level:(query:ERROR,type:phrase)))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'96f67170-0c49-11e9-85c1-1d63df8c862c',key:extra.key,negate:!f,params:(query:US-CAL-CISO,type:phrase),type:phrase,value:US-CAL-CISO),query:(match:(extra.key:(query:US-CAL-CISO,type:phrase))))),index:'96f67170-0c49-11e9-85c1-1d63df8c862c',interval:auto,query:(language:lucene,query:''),sort:!('@timestamp',desc))):\r\n>invalid point: {'zoneKey': 'US-CAL-CISO', 'production': defaultdict(<class 'float'>, {'solar': 0.0, 'wind': 35.0, 'geothermal': 537.0, 'biomass': 385.0, 'hydro': 916.0, 'coal': 16.0, 'nuclear': -34.0, 'gas': 10688.0, 'unknown': 0.0}), 'storage': defaultdict(<class 'float'>, {'battery': 3.0}), 'source': 'caiso.com', 'datetime': datetime.datetime(2020, 10, 28, 4, 10, tzinfo=tzfile('/usr/share/zoneinfo/US/Pacific')), 'schemaVersion': 3}, reason:US-CAL-CISO: key nuclear has negative value -34.0\r\n\r\nGiven the small value and that it's zero on the [CAISO web site](http://www.caiso.com/TodaysOutlook/Pages/supply.aspx), this is probably self consumption. Some googling shows that both units of Diablo Canyon, California's sole nuclear power plant, are [down for maintainance](https://keyt.com/news/san-luis-obispo-county/2020/10/15/pge-removes-diablo-canyons-unit-2-for-unexpected-maintenance/).\n", "before_files": [{"content": "#!/usr/bin/env python3\n\nimport arrow\nimport pandas\nimport requests\nfrom bs4 import BeautifulSoup\nfrom collections import defaultdict\n\nFUEL_SOURCE_CSV = 'http://www.caiso.com/outlook/SP/fuelsource.csv'\n\nMX_EXCHANGE_URL = 'http://www.cenace.gob.mx/Paginas/Publicas/Info/DemandaRegional.aspx'\n\ndef fetch_production(zone_key='US-CA', session=None, target_datetime=None,\n                     logger=None):\n    \"\"\"Requests the last known production mix (in MW) of a given country\n\n    Arguments:\n    zone_key: used in case a parser is able to fetch multiple countries\n    session: request session passed in order to re-use an existing session\n\n    Return:\n    A dictionary in the form:\n    {\n      'zoneKey': 'FR',\n      'datetime': '2017-01-01T00:00:00Z',\n      'production': {\n          'biomass': 0.0,\n          'coal': 0.0,\n          'gas': 0.0,\n          'hydro': 0.0,\n          'nuclear': null,\n          'oil': 0.0,\n          'solar': 0.0,\n          'wind': 0.0,\n          'geothermal': 0.0,\n          'unknown': 0.0\n      },\n      'storage': {\n          'hydro': -10.0,\n      },\n      'source': 'mysource.com'\n    }\n    \"\"\"\n    if target_datetime:\n        return fetch_historical_production(target_datetime, zone_key)\n\n    target_datetime = arrow.get(target_datetime)\n\n    # Get the production from the CSV\n    csv = pandas.read_csv(FUEL_SOURCE_CSV)\n    latest_index = len(csv) - 1\n    production_map = {\n        'Solar': 'solar',\n        'Wind': 'wind',\n        'Geothermal': 'geothermal',\n        'Biomass': 'biomass',\n        'Biogas': 'biomass',\n        'Small hydro': 'hydro',\n        'Coal': 'coal',\n        'Nuclear': 'nuclear',\n        'Natural gas': 'gas',\n        'Large hydro': 'hydro',\n        'Other': 'unknown'\n    }\n    storage_map = {\n        'Batteries': 'battery'\n    }\n    daily_data = []\n    for i in range(0, latest_index + 1):\n        h, m = map(int, csv['Time'][i].split(':'))\n        date = arrow.utcnow().to('US/Pacific').replace(hour=h, minute=m,\n                                                       second=0, microsecond=0)\n        data = {\n            'zoneKey': zone_key,\n            'production': defaultdict(float),\n            'storage': defaultdict(float),\n            'source': 'caiso.com',\n            'datetime': date.datetime\n        }\n\n        # map items from names in CAISO CSV to names used in Electricity Map\n        for ca_gen_type, mapped_gen_type in production_map.items():\n            production = float(csv[ca_gen_type][i])\n            \n            if mapped_gen_type == 'solar' and production < 0:\n                logger.warn('Solar production for US_CA was reported as less than 0 and was clamped')\n                production = 0.0\n            \n            # if another mean of production created a value, sum them up\n            data['production'][mapped_gen_type] += production\n\n        for ca_storage_type, mapped_storage_type in storage_map.items():\n            storage = -float(csv[ca_storage_type][i])\n\n            # if another mean of storage created a value, sum them up\n            data['storage'][mapped_storage_type] += storage\n\n        daily_data.append(data)\n\n    return daily_data\n\n\ndef fetch_historical_production(target_datetime, zone_key):\n    return fetch_historical_data(target_datetime, zone_key)[0]\n\n\ndef fetch_historical_exchange(target_datetime):\n    return fetch_historical_data(target_datetime)[1]\n\n\ndef fetch_historical_data(target_datetime, zone_key='US-CA'):\n    # caiso.com provides daily data until the day before today\n    # get a clean date at the beginning of yesterday\n    target_date = arrow.get(target_datetime).to('US/Pacific').replace(\n        hour=0, minute=0, second=0, microsecond=0)\n\n    url = 'http://content.caiso.com/green/renewrpt/' + target_date.format(\n        'YYYYMMDD') + '_DailyRenewablesWatch.txt'\n\n    renewable_resources = pandas.read_table(\n        url, sep='\\t\\t', skiprows=2, header=None,\n        names=['Hour', 'GEOTHERMAL', 'BIOMASS', 'BIOGAS', 'SMALL HYDRO',\n               'WIND TOTAL', 'SOLAR PV', 'SOLAR THERMAL'],\n        skipfooter=27, skipinitialspace=True, engine='python')\n    other_resources = pandas.read_table(\n        url, sep='\\t\\t', skiprows=30, header=None,\n        names=['Hour', 'RENEWABLES', 'NUCLEAR', 'THERMAL', 'IMPORTS', 'HYDRO'],\n        skipinitialspace=True, engine='python')\n\n    daily_data, import_data = [], []\n\n    for i in range(0, 24):\n        daily_data.append({\n            'zoneKey': zone_key,\n            'storage': {},\n            'source': 'caiso.com',\n            'production': {\n                'biomass': float(renewable_resources['BIOMASS'][i]),\n                'gas': float(renewable_resources['BIOGAS'][i])\n                       + float(other_resources['THERMAL'][i]),\n                'hydro': float(renewable_resources['SMALL HYDRO'][i])\n                         + float(other_resources['HYDRO'][i]),\n                'nuclear': float(other_resources['NUCLEAR'][i]),\n                'solar': float(renewable_resources['SOLAR PV'][i])\n                         + float(renewable_resources['SOLAR THERMAL'][i]),\n                'wind': float(renewable_resources['WIND TOTAL'][i]),\n                'geothermal': float(renewable_resources['GEOTHERMAL'][i]),\n            },\n            'datetime': target_date.shift(hours=i + 1).datetime,\n        })\n        import_data.append(\n            {\n                'sortedZoneKeys': 'US->US-CA',\n                'datetime': target_date.shift(hours=i + 1).datetime,\n                'netFlow': float(other_resources['IMPORTS'][i]),\n                'source': 'caiso.com'\n            }\n        )\n\n    return daily_data, import_data\n\n\ndef fetch_MX_exchange(s):\n    req = s.get(MX_EXCHANGE_URL)\n    soup = BeautifulSoup(req.text, 'html.parser')\n    exchange_div = soup.find(\"div\", attrs={'id': 'IntercambioUSA-BCA'})\n    val = exchange_div.text\n\n    # cenace html uses unicode hyphens instead of minus signs\n    try:\n        val = val.replace(chr(8208), chr(45))\n    except ValueError:\n        pass\n\n    # negative value indicates flow from CA to MX\n\n    return float(val)\n\n\ndef fetch_exchange(zone_key1, zone_key2, session=None, target_datetime=None,\n                   logger=None):\n    \"\"\"Requests the last known power exchange (in MW) between two zones\n    Arguments:\n    zone_key1: the first country code\n    zone_key2: the second country code; order of the two codes in params\n      doesn't matter\n    session: request session passed in order to re-use an existing session\n    Return:\n    A dictionary in the form:\n    {\n      'sortedZoneKeys': 'DK->NO',\n      'datetime': '2017-01-01T00:00:00Z',\n      'netFlow': 0.0,\n      'source': 'mysource.com'\n    }\n    where net flow is from DK into NO\n    \"\"\"\n    sorted_zone_keys = '->'.join(sorted([zone_key1, zone_key2]))\n\n    s = session or requests.Session()\n\n    if sorted_zone_keys == 'MX-BC->US-CA' or sorted_zone_keys == 'MX-BC->US-CAL-CISO':\n        netflow = fetch_MX_exchange(s)\n        exchange = {\n          'sortedZoneKeys': sorted_zone_keys,\n          'datetime': arrow.now('America/Tijuana').datetime,\n          'netFlow': netflow,\n          'source': 'cenace.gob.mx'\n        }\n        return exchange\n\n    if target_datetime:\n        return fetch_historical_exchange(target_datetime)\n\n    # CSV has imports to California as positive.\n    # Electricity Map expects A->B to indicate flow to B as positive.\n    # So values in CSV can be used as-is.\n\n    csv = pandas.read_csv(FUEL_SOURCE_CSV)\n    latest_index = len(csv) - 1\n    daily_data = []\n    for i in range(0, latest_index + 1):\n        h, m = map(int, csv['Time'][i].split(':'))\n        date = arrow.utcnow().to('US/Pacific').replace(hour=h, minute=m,\n                                                       second=0, microsecond=0)\n        data = {\n            'sortedZoneKeys': sorted_zone_keys,\n            'datetime': date.datetime,\n            'netFlow': float(csv['Imports'][i]),\n            'source': 'caiso.com'\n        }\n\n        daily_data.append(data)\n\n    return daily_data\n\n\nif __name__ == '__main__':\n    \"Main method, not used by Electricity Map backend, but handy for testing\"\n\n    from pprint import pprint\n\n    print('fetch_production() ->')\n    pprint(fetch_production())\n\n    print('fetch_exchange(\"US-CA\", \"US\") ->')\n    #pprint(fetch_exchange(\"US-CA\", \"US\"))\n\n    print('fetch_exchange(\"MX-BC\", \"US-CA\")')\n    pprint(fetch_exchange(\"MX-BC\", \"US-CA\"))\n", "path": "parsers/US_CA.py"}], "after_files": [{"content": "#!/usr/bin/env python3\n\nimport arrow\nimport pandas\nimport requests\nfrom bs4 import BeautifulSoup\nfrom collections import defaultdict\nimport logging\n\nFUEL_SOURCE_CSV = 'http://www.caiso.com/outlook/SP/fuelsource.csv'\n\nMX_EXCHANGE_URL = 'http://www.cenace.gob.mx/Paginas/Publicas/Info/DemandaRegional.aspx'\n\ndef fetch_production(zone_key='US-CA', session=None, target_datetime=None,\n                     logger: logging.Logger = logging.getLogger(__name__)):\n    \"\"\"Requests the last known production mix (in MW) of a given country\n\n    Arguments:\n    zone_key: used in case a parser is able to fetch multiple countries\n    session: request session passed in order to re-use an existing session\n\n    Return:\n    A dictionary in the form:\n    {\n      'zoneKey': 'FR',\n      'datetime': '2017-01-01T00:00:00Z',\n      'production': {\n          'biomass': 0.0,\n          'coal': 0.0,\n          'gas': 0.0,\n          'hydro': 0.0,\n          'nuclear': null,\n          'oil': 0.0,\n          'solar': 0.0,\n          'wind': 0.0,\n          'geothermal': 0.0,\n          'unknown': 0.0\n      },\n      'storage': {\n          'hydro': -10.0,\n      },\n      'source': 'mysource.com'\n    }\n    \"\"\"\n    if target_datetime:\n        return fetch_historical_production(target_datetime, zone_key)\n\n    target_datetime = arrow.get(target_datetime)\n\n    # Get the production from the CSV\n    csv = pandas.read_csv(FUEL_SOURCE_CSV)\n    latest_index = len(csv) - 1\n    production_map = {\n        'Solar': 'solar',\n        'Wind': 'wind',\n        'Geothermal': 'geothermal',\n        'Biomass': 'biomass',\n        'Biogas': 'biomass',\n        'Small hydro': 'hydro',\n        'Coal': 'coal',\n        'Nuclear': 'nuclear',\n        'Natural gas': 'gas',\n        'Large hydro': 'hydro',\n        'Other': 'unknown'\n    }\n    storage_map = {\n        'Batteries': 'battery'\n    }\n    daily_data = []\n    for i in range(0, latest_index + 1):\n        h, m = map(int, csv['Time'][i].split(':'))\n        date = arrow.utcnow().to('US/Pacific').replace(hour=h, minute=m,\n                                                       second=0, microsecond=0)\n        data = {\n            'zoneKey': zone_key,\n            'production': defaultdict(float),\n            'storage': defaultdict(float),\n            'source': 'caiso.com',\n            'datetime': date.datetime\n        }\n\n        # map items from names in CAISO CSV to names used in Electricity Map\n        for ca_gen_type, mapped_gen_type in production_map.items():\n            production = float(csv[ca_gen_type][i])\n            \n            if production < 0 and (mapped_gen_type == 'solar' or mapped_gen_type == 'nuclear'):\n                logger.warn(ca_gen_type + ' production for US_CA was reported as less than 0 and was clamped')\n                production = 0.0\n            \n            # if another mean of production created a value, sum them up\n            data['production'][mapped_gen_type] += production\n\n        for ca_storage_type, mapped_storage_type in storage_map.items():\n            storage = -float(csv[ca_storage_type][i])\n\n            # if another mean of storage created a value, sum them up\n            data['storage'][mapped_storage_type] += storage\n\n        daily_data.append(data)\n\n    return daily_data\n\n\ndef fetch_historical_production(target_datetime, zone_key):\n    return fetch_historical_data(target_datetime, zone_key)[0]\n\n\ndef fetch_historical_exchange(target_datetime):\n    return fetch_historical_data(target_datetime)[1]\n\n\ndef fetch_historical_data(target_datetime, zone_key='US-CA'):\n    # caiso.com provides daily data until the day before today\n    # get a clean date at the beginning of yesterday\n    target_date = arrow.get(target_datetime).to('US/Pacific').replace(\n        hour=0, minute=0, second=0, microsecond=0)\n\n    url = 'http://content.caiso.com/green/renewrpt/' + target_date.format(\n        'YYYYMMDD') + '_DailyRenewablesWatch.txt'\n\n    renewable_resources = pandas.read_table(\n        url, sep='\\t\\t', skiprows=2, header=None,\n        names=['Hour', 'GEOTHERMAL', 'BIOMASS', 'BIOGAS', 'SMALL HYDRO',\n               'WIND TOTAL', 'SOLAR PV', 'SOLAR THERMAL'],\n        skipfooter=27, skipinitialspace=True, engine='python')\n    other_resources = pandas.read_table(\n        url, sep='\\t\\t', skiprows=30, header=None,\n        names=['Hour', 'RENEWABLES', 'NUCLEAR', 'THERMAL', 'IMPORTS', 'HYDRO'],\n        skipinitialspace=True, engine='python')\n\n    daily_data, import_data = [], []\n\n    for i in range(0, 24):\n        daily_data.append({\n            'zoneKey': zone_key,\n            'storage': {},\n            'source': 'caiso.com',\n            'production': {\n                'biomass': float(renewable_resources['BIOMASS'][i]),\n                'gas': float(renewable_resources['BIOGAS'][i])\n                       + float(other_resources['THERMAL'][i]),\n                'hydro': float(renewable_resources['SMALL HYDRO'][i])\n                         + float(other_resources['HYDRO'][i]),\n                'nuclear': float(other_resources['NUCLEAR'][i]),\n                'solar': float(renewable_resources['SOLAR PV'][i])\n                         + float(renewable_resources['SOLAR THERMAL'][i]),\n                'wind': float(renewable_resources['WIND TOTAL'][i]),\n                'geothermal': float(renewable_resources['GEOTHERMAL'][i]),\n            },\n            'datetime': target_date.shift(hours=i + 1).datetime,\n        })\n        import_data.append(\n            {\n                'sortedZoneKeys': 'US->US-CA',\n                'datetime': target_date.shift(hours=i + 1).datetime,\n                'netFlow': float(other_resources['IMPORTS'][i]),\n                'source': 'caiso.com'\n            }\n        )\n\n    return daily_data, import_data\n\n\ndef fetch_MX_exchange(s):\n    req = s.get(MX_EXCHANGE_URL)\n    soup = BeautifulSoup(req.text, 'html.parser')\n    exchange_div = soup.find(\"div\", attrs={'id': 'IntercambioUSA-BCA'})\n    val = exchange_div.text\n\n    # cenace html uses unicode hyphens instead of minus signs\n    try:\n        val = val.replace(chr(8208), chr(45))\n    except ValueError:\n        pass\n\n    # negative value indicates flow from CA to MX\n\n    return float(val)\n\n\ndef fetch_exchange(zone_key1, zone_key2, session=None, target_datetime=None,\n                   logger=None):\n    \"\"\"Requests the last known power exchange (in MW) between two zones\n    Arguments:\n    zone_key1: the first country code\n    zone_key2: the second country code; order of the two codes in params\n      doesn't matter\n    session: request session passed in order to re-use an existing session\n    Return:\n    A dictionary in the form:\n    {\n      'sortedZoneKeys': 'DK->NO',\n      'datetime': '2017-01-01T00:00:00Z',\n      'netFlow': 0.0,\n      'source': 'mysource.com'\n    }\n    where net flow is from DK into NO\n    \"\"\"\n    sorted_zone_keys = '->'.join(sorted([zone_key1, zone_key2]))\n\n    s = session or requests.Session()\n\n    if sorted_zone_keys == 'MX-BC->US-CA' or sorted_zone_keys == 'MX-BC->US-CAL-CISO':\n        netflow = fetch_MX_exchange(s)\n        exchange = {\n          'sortedZoneKeys': sorted_zone_keys,\n          'datetime': arrow.now('America/Tijuana').datetime,\n          'netFlow': netflow,\n          'source': 'cenace.gob.mx'\n        }\n        return exchange\n\n    if target_datetime:\n        return fetch_historical_exchange(target_datetime)\n\n    # CSV has imports to California as positive.\n    # Electricity Map expects A->B to indicate flow to B as positive.\n    # So values in CSV can be used as-is.\n\n    csv = pandas.read_csv(FUEL_SOURCE_CSV)\n    latest_index = len(csv) - 1\n    daily_data = []\n    for i in range(0, latest_index + 1):\n        h, m = map(int, csv['Time'][i].split(':'))\n        date = arrow.utcnow().to('US/Pacific').replace(hour=h, minute=m,\n                                                       second=0, microsecond=0)\n        data = {\n            'sortedZoneKeys': sorted_zone_keys,\n            'datetime': date.datetime,\n            'netFlow': float(csv['Imports'][i]),\n            'source': 'caiso.com'\n        }\n\n        daily_data.append(data)\n\n    return daily_data\n\n\nif __name__ == '__main__':\n    \"Main method, not used by Electricity Map backend, but handy for testing\"\n\n    from pprint import pprint\n\n    print('fetch_production() ->')\n    pprint(fetch_production())\n\n    print('fetch_exchange(\"US-CA\", \"US\") ->')\n    #pprint(fetch_exchange(\"US-CA\", \"US\"))\n\n    print('fetch_exchange(\"MX-BC\", \"US-CA\")')\n    pprint(fetch_exchange(\"MX-BC\", \"US-CA\"))\n", "path": "parsers/US_CA.py"}]}