prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Add JP prices
  See https://github.com/tmrowco/electricitymap-contrib/pull/1543#issuecomment-411281685 by @tmslaine 
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `parsers/JP.py`
  Content:
  ```
  1 #!/usr/bin/env python3
  2 # coding=utf-8
  3 import logging
  4 # The arrow library is used to handle datetimes
  5 import arrow
  6 import pandas as pd
  7 from . import occtonet
  8 
  9 # Abbreviations
  10 # JP-HKD : Hokkaido
  11 # JP-TH  : Tohoku
  12 # JP-TK  : Tokyo area
  13 # JP-CB  : Chubu
  14 # JP-HR  : Hokuriku
  15 # JP-KN  : Kansai
  16 # JP-SK  : Shikoku
  17 # JP-KY  : Kyushu
  18 # JP-ON  : Okinawa
  19 
  20 def fetch_production(zone_key='JP-TK', session=None, target_datetime=None,
  21                      logger=logging.getLogger(__name__)):
  22     """
  23     Calculates production from consumption and imports for a given area
  24     All production is mapped to unknown
  25     """
  26     if target_datetime:
  27         raise NotImplementedError(
  28             'This parser is not yet able to parse past dates')
  29     exch_map = {
  30         'JP-HKD':['JP-TH'],
  31         'JP-TH':['JP-TK'],
  32         'JP-TK':['JP-TH', 'JP-CB'],
  33         'JP-CB':['JP-TK', 'JP-HR', 'JP-KN'],
  34         'JP-HR':['JP-CB', 'JP-KN'],
  35         'JP-KN':['JP-CB', 'JP-HR', 'JP-SK', 'JP-CG'],
  36         'JP-SK':['JP-KN', 'JP-CG'],
  37         'JP-CG':['JP-KN', 'JP-SK', 'JP-KY']
  38         }
  39     df = fetch_consumption_df(zone_key, target_datetime)
  40     df['imports'] = 0
  41     for zone in exch_map[zone_key]:
  42         df2 = occtonet.fetch_exchange(zone_key, zone, target_datetime)
  43         df2 = pd.DataFrame(df2)
  44         exchname = df2.loc[0, 'sortedZoneKeys']
  45         df2 = df2[['datetime', 'netFlow']]
  46         df2.columns = ['datetime', exchname]
  47         df = pd.merge(df, df2, how='inner', on='datetime')
  48         if exchname.split('->')[-1] == zone_key:
  49             df['imports'] = df['imports']+df[exchname]
  50         else:
  51             df['imports'] = df['imports']-df[exchname]
  52     df['prod'] = df['cons']-df['imports']
  53     df = df[['datetime', 'prod']]
  54     # add a row to production for each entry in the dictionary:
  55     sources = {
  56         'JP-HKD':'denkiyoho.hepco.co.jp',
  57         'JP-TH':'setsuden.tohoku-epco.co.jp',
  58         'JP-TK':'www.tepco.co.jp',
  59         'JP-CB':'denki-yoho.chuden.jp',
  60         'JP-HR':'www.rikuden.co.jp/denki-yoho',
  61         'JP-KN':'www.kepco.co.jp',
  62         'JP-SK':'www.energia.co.jp',
  63         'JP-CG':'www.yonden.co.jp'
  64         }
  65     datalist = []
  66     for i in range(df.shape[0]):
  67         data = {
  68             'zoneKey': zone_key,
  69             'datetime': df.loc[i, 'datetime'].to_pydatetime(),
  70             'production': {
  71                 'biomass': None,
  72                 'coal': None,
  73                 'gas': None,
  74                 'hydro': None,
  75                 'nuclear': None,
  76                 'oil': None,
  77                 'solar': None,
  78                 'wind': None,
  79                 'geothermal': None,
  80                 'unknown': df.loc[i, 'prod']
  81             },
  82             'storage': {},
  83             'source': ['occtonet.or.jp', sources[zone_key]]
  84             }
  85         datalist.append(data)
  86     return datalist
  87 
  88 
  89 def fetch_consumption_df(zone_key='JP-TK', target_datetime=None,
  90                       logger=logging.getLogger(__name__)):
  91     """
  92     Returns the consumption for an area as a pandas DataFrame
  93     """
  94     datestamp = arrow.get(target_datetime).to('Asia/Tokyo').strftime('%Y%m%d')
  95     consumption_url = {
  96         'JP-HKD': 'http://denkiyoho.hepco.co.jp/area/data/juyo_01_{}.csv'.format(datestamp),
  97         'JP-TH': 'http://setsuden.tohoku-epco.co.jp/common/demand/juyo_02_{}.csv'.format(datestamp),
  98         'JP-TK': 'http://www.tepco.co.jp/forecast/html/images/juyo-j.csv',
  99         'JP-HR': 'http://www.rikuden.co.jp/denki-yoho/csv/juyo_05_{}.csv'.format(datestamp),
  100         'JP-CB': 'http://denki-yoho.chuden.jp/denki_yoho_content_data/juyo_cepco003.csv',
  101         'JP-KN': 'http://www.kepco.co.jp/yamasou/juyo1_kansai.csv',
  102         'JP-CG': 'http://www.energia.co.jp/jukyuu/sys/juyo_07_{}.csv'.format(datestamp),
  103         'JP-SK': 'http://www.yonden.co.jp/denkiyoho/juyo_shikoku.csv'
  104         }
  105     # First roughly 40 rows of the consumption files have hourly data,
  106     # the parser skips to the rows with 5-min actual values 
  107     if zone_key == 'JP-KN':
  108         startrow = 44
  109     else:
  110         startrow = 42
  111     df = pd.read_csv(consumption_url[zone_key], skiprows=list(range(startrow)),
  112                      encoding='shift-jis')
  113     df.columns = ['Date', 'Time', 'cons']
  114     # Convert 万kW to MW
  115     df['cons'] = 10*df['cons']
  116     df = df.dropna()
  117     df['datetime'] = df.apply(parse_dt, axis=1)
  118     df = df[['datetime', 'cons']]
  119     return df
  120 
  121 def parse_dt(row):
  122     """
  123     Parses timestamps from date and time
  124     """
  125     return arrow.get(' '.join([row['Date'], row['Time']]).replace('/', '-'),
  126                      'YYYY-M-D H:mm').replace(tzinfo='Asia/Tokyo').datetime
  127 
  128 if __name__ == '__main__':
  129     """Main method, never used by the Electricity Map backend, but handy for testing."""
  130 
  131     print('fetch_production() ->')
  132     print(fetch_production())
  133
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/parsers/JP.py b/parsers/JP.py
  --- a/parsers/JP.py
  +++ b/parsers/JP.py
  @@ -3,6 +3,7 @@
   import logging
   # The arrow library is used to handle datetimes
   import arrow
  +import datetime as dt
   import pandas as pd
   from . import occtonet
   
  @@ -16,6 +17,8 @@
   # JP-SK  : Shikoku
   # JP-KY  : Kyushu
   # JP-ON  : Okinawa
  +# JP-CG  : Chūgoku
  +
   
   def fetch_production(zone_key='JP-TK', session=None, target_datetime=None,
                        logger=logging.getLogger(__name__)):
  @@ -118,6 +121,47 @@
       df = df[['datetime', 'cons']]
       return df
   
  +
  +def fetch_price(zone_key='JP-TK', session=None, target_datetime=None,
  +                logger=logging.getLogger(__name__)):
  +    if target_datetime is None:
  +        target_datetime = dt.datetime.now() + dt.timedelta(days=1)
  +
  +    # price files contain data for fiscal year and not calendar year.
  +    if target_datetime.month <= 3:
  +        fiscal_year = target_datetime.year - 1
  +    else:
  +        fiscal_year = target_datetime.year
  +    url = 'http://www.jepx.org/market/excel/spot_{}.csv'.format(fiscal_year)
  +    df = pd.read_csv(url)
  +
  +    df = df.iloc[:, [0, 1, 6, 7, 8, 9, 10, 11, 12, 13, 14]]
  +    df.columns = ['Date', 'Period', 'JP-HKD', 'JP-TH', 'JP-TK', 'JP-CB',
  +                  'JP-HR', 'JP-KN', 'JP-CG', 'JP-SK', 'JP-KY']
  +
  +    if zone_key not in df.columns[2:]:
  +        return []
  +
  +    start = target_datetime - dt.timedelta(days=1)
  +    df['Date'] = df['Date'].apply(lambda x: dt.datetime.strptime(x, '%Y/%m/%d'))
  +    df = df[(df['Date'] >= start.date()) & (df['Date'] <= target_datetime.date())]
  +
  +    df['datetime'] = df.apply(lambda row: arrow.get(row['Date']).shift(
  +        minutes=30 * (row['Period'] - 1)).replace(tzinfo='Asia/Tokyo'), axis=1)
  +
  +    data = list()
  +    for row in df.iterrows():
  +        data.append({
  +            'zoneKey': zone_key,
  +            'currency': 'JPY',
  +            'datetime': row[1]['datetime'].datetime,
  +            'price': row[1][zone_key],
  +            'source': 'jepx.org'
  +        })
  +
  +    return data
  +
  +
   def parse_dt(row):
       """
       Parses timestamps from date and time
  @@ -125,8 +169,11 @@
       return arrow.get(' '.join([row['Date'], row['Time']]).replace('/', '-'),
                        'YYYY-M-D H:mm').replace(tzinfo='Asia/Tokyo').datetime
   
  +
   if __name__ == '__main__':
       """Main method, never used by the Electricity Map backend, but handy for testing."""
   
       print('fetch_production() ->')
       print(fetch_production())
  +    print('fetch_price() ->')
  +    print(fetch_price())
  
verification_info: {"golden_diff": "diff --git a/parsers/JP.py b/parsers/JP.py\n--- a/parsers/JP.py\n+++ b/parsers/JP.py\n@@ -3,6 +3,7 @@\n import logging\n # The arrow library is used to handle datetimes\n import arrow\n+import datetime as dt\n import pandas as pd\n from . import occtonet\n \n@@ -16,6 +17,8 @@\n # JP-SK  : Shikoku\n # JP-KY  : Kyushu\n # JP-ON  : Okinawa\n+# JP-CG  : Ch\u016bgoku\n+\n \n def fetch_production(zone_key='JP-TK', session=None, target_datetime=None,\n                      logger=logging.getLogger(__name__)):\n@@ -118,6 +121,47 @@\n     df = df[['datetime', 'cons']]\n     return df\n \n+\n+def fetch_price(zone_key='JP-TK', session=None, target_datetime=None,\n+                logger=logging.getLogger(__name__)):\n+    if target_datetime is None:\n+        target_datetime = dt.datetime.now() + dt.timedelta(days=1)\n+\n+    # price files contain data for fiscal year and not calendar year.\n+    if target_datetime.month <= 3:\n+        fiscal_year = target_datetime.year - 1\n+    else:\n+        fiscal_year = target_datetime.year\n+    url = 'http://www.jepx.org/market/excel/spot_{}.csv'.format(fiscal_year)\n+    df = pd.read_csv(url)\n+\n+    df = df.iloc[:, [0, 1, 6, 7, 8, 9, 10, 11, 12, 13, 14]]\n+    df.columns = ['Date', 'Period', 'JP-HKD', 'JP-TH', 'JP-TK', 'JP-CB',\n+                  'JP-HR', 'JP-KN', 'JP-CG', 'JP-SK', 'JP-KY']\n+\n+    if zone_key not in df.columns[2:]:\n+        return []\n+\n+    start = target_datetime - dt.timedelta(days=1)\n+    df['Date'] = df['Date'].apply(lambda x: dt.datetime.strptime(x, '%Y/%m/%d'))\n+    df = df[(df['Date'] >= start.date()) & (df['Date'] <= target_datetime.date())]\n+\n+    df['datetime'] = df.apply(lambda row: arrow.get(row['Date']).shift(\n+        minutes=30 * (row['Period'] - 1)).replace(tzinfo='Asia/Tokyo'), axis=1)\n+\n+    data = list()\n+    for row in df.iterrows():\n+        data.append({\n+            'zoneKey': zone_key,\n+            'currency': 'JPY',\n+            'datetime': row[1]['datetime'].datetime,\n+            'price': row[1][zone_key],\n+            'source': 'jepx.org'\n+        })\n+\n+    return data\n+\n+\n def parse_dt(row):\n     \"\"\"\n     Parses timestamps from date and time\n@@ -125,8 +169,11 @@\n     return arrow.get(' '.join([row['Date'], row['Time']]).replace('/', '-'),\n                      'YYYY-M-D H:mm').replace(tzinfo='Asia/Tokyo').datetime\n \n+\n if __name__ == '__main__':\n     \"\"\"Main method, never used by the Electricity Map backend, but handy for testing.\"\"\"\n \n     print('fetch_production() ->')\n     print(fetch_production())\n+    print('fetch_price() ->')\n+    print(fetch_price())\n", "issue": "Add JP prices\nSee https://github.com/tmrowco/electricitymap-contrib/pull/1543#issuecomment-411281685 by @tmslaine \n", "before_files": [{"content": "#!/usr/bin/env python3\n# coding=utf-8\nimport logging\n# The arrow library is used to handle datetimes\nimport arrow\nimport pandas as pd\nfrom . import occtonet\n\n# Abbreviations\n# JP-HKD : Hokkaido\n# JP-TH  : Tohoku\n# JP-TK  : Tokyo area\n# JP-CB  : Chubu\n# JP-HR  : Hokuriku\n# JP-KN  : Kansai\n# JP-SK  : Shikoku\n# JP-KY  : Kyushu\n# JP-ON  : Okinawa\n\ndef fetch_production(zone_key='JP-TK', session=None, target_datetime=None,\n                     logger=logging.getLogger(__name__)):\n    \"\"\"\n    Calculates production from consumption and imports for a given area\n    All production is mapped to unknown\n    \"\"\"\n    if target_datetime:\n        raise NotImplementedError(\n            'This parser is not yet able to parse past dates')\n    exch_map = {\n        'JP-HKD':['JP-TH'],\n        'JP-TH':['JP-TK'],\n        'JP-TK':['JP-TH', 'JP-CB'],\n        'JP-CB':['JP-TK', 'JP-HR', 'JP-KN'],\n        'JP-HR':['JP-CB', 'JP-KN'],\n        'JP-KN':['JP-CB', 'JP-HR', 'JP-SK', 'JP-CG'],\n        'JP-SK':['JP-KN', 'JP-CG'],\n        'JP-CG':['JP-KN', 'JP-SK', 'JP-KY']\n        }\n    df = fetch_consumption_df(zone_key, target_datetime)\n    df['imports'] = 0\n    for zone in exch_map[zone_key]:\n        df2 = occtonet.fetch_exchange(zone_key, zone, target_datetime)\n        df2 = pd.DataFrame(df2)\n        exchname = df2.loc[0, 'sortedZoneKeys']\n        df2 = df2[['datetime', 'netFlow']]\n        df2.columns = ['datetime', exchname]\n        df = pd.merge(df, df2, how='inner', on='datetime')\n        if exchname.split('->')[-1] == zone_key:\n            df['imports'] = df['imports']+df[exchname]\n        else:\n            df['imports'] = df['imports']-df[exchname]\n    df['prod'] = df['cons']-df['imports']\n    df = df[['datetime', 'prod']]\n    # add a row to production for each entry in the dictionary:\n    sources = {\n        'JP-HKD':'denkiyoho.hepco.co.jp',\n        'JP-TH':'setsuden.tohoku-epco.co.jp',\n        'JP-TK':'www.tepco.co.jp',\n        'JP-CB':'denki-yoho.chuden.jp',\n        'JP-HR':'www.rikuden.co.jp/denki-yoho',\n        'JP-KN':'www.kepco.co.jp',\n        'JP-SK':'www.energia.co.jp',\n        'JP-CG':'www.yonden.co.jp'\n        }\n    datalist = []\n    for i in range(df.shape[0]):\n        data = {\n            'zoneKey': zone_key,\n            'datetime': df.loc[i, 'datetime'].to_pydatetime(),\n            'production': {\n                'biomass': None,\n                'coal': None,\n                'gas': None,\n                'hydro': None,\n                'nuclear': None,\n                'oil': None,\n                'solar': None,\n                'wind': None,\n                'geothermal': None,\n                'unknown': df.loc[i, 'prod']\n            },\n            'storage': {},\n            'source': ['occtonet.or.jp', sources[zone_key]]\n            }\n        datalist.append(data)\n    return datalist\n\n\ndef fetch_consumption_df(zone_key='JP-TK', target_datetime=None,\n                      logger=logging.getLogger(__name__)):\n    \"\"\"\n    Returns the consumption for an area as a pandas DataFrame\n    \"\"\"\n    datestamp = arrow.get(target_datetime).to('Asia/Tokyo').strftime('%Y%m%d')\n    consumption_url = {\n        'JP-HKD': 'http://denkiyoho.hepco.co.jp/area/data/juyo_01_{}.csv'.format(datestamp),\n        'JP-TH': 'http://setsuden.tohoku-epco.co.jp/common/demand/juyo_02_{}.csv'.format(datestamp),\n        'JP-TK': 'http://www.tepco.co.jp/forecast/html/images/juyo-j.csv',\n        'JP-HR': 'http://www.rikuden.co.jp/denki-yoho/csv/juyo_05_{}.csv'.format(datestamp),\n        'JP-CB': 'http://denki-yoho.chuden.jp/denki_yoho_content_data/juyo_cepco003.csv',\n        'JP-KN': 'http://www.kepco.co.jp/yamasou/juyo1_kansai.csv',\n        'JP-CG': 'http://www.energia.co.jp/jukyuu/sys/juyo_07_{}.csv'.format(datestamp),\n        'JP-SK': 'http://www.yonden.co.jp/denkiyoho/juyo_shikoku.csv'\n        }\n    # First roughly 40 rows of the consumption files have hourly data,\n    # the parser skips to the rows with 5-min actual values \n    if zone_key == 'JP-KN':\n        startrow = 44\n    else:\n        startrow = 42\n    df = pd.read_csv(consumption_url[zone_key], skiprows=list(range(startrow)),\n                     encoding='shift-jis')\n    df.columns = ['Date', 'Time', 'cons']\n    # Convert \u4e07kW to MW\n    df['cons'] = 10*df['cons']\n    df = df.dropna()\n    df['datetime'] = df.apply(parse_dt, axis=1)\n    df = df[['datetime', 'cons']]\n    return df\n\ndef parse_dt(row):\n    \"\"\"\n    Parses timestamps from date and time\n    \"\"\"\n    return arrow.get(' '.join([row['Date'], row['Time']]).replace('/', '-'),\n                     'YYYY-M-D H:mm').replace(tzinfo='Asia/Tokyo').datetime\n\nif __name__ == '__main__':\n    \"\"\"Main method, never used by the Electricity Map backend, but handy for testing.\"\"\"\n\n    print('fetch_production() ->')\n    print(fetch_production())\n", "path": "parsers/JP.py"}], "after_files": [{"content": "#!/usr/bin/env python3\n# coding=utf-8\nimport logging\n# The arrow library is used to handle datetimes\nimport arrow\nimport datetime as dt\nimport pandas as pd\nfrom . import occtonet\n\n# Abbreviations\n# JP-HKD : Hokkaido\n# JP-TH  : Tohoku\n# JP-TK  : Tokyo area\n# JP-CB  : Chubu\n# JP-HR  : Hokuriku\n# JP-KN  : Kansai\n# JP-SK  : Shikoku\n# JP-KY  : Kyushu\n# JP-ON  : Okinawa\n# JP-CG  : Ch\u016bgoku\n\n\ndef fetch_production(zone_key='JP-TK', session=None, target_datetime=None,\n                     logger=logging.getLogger(__name__)):\n    \"\"\"\n    Calculates production from consumption and imports for a given area\n    All production is mapped to unknown\n    \"\"\"\n    if target_datetime:\n        raise NotImplementedError(\n            'This parser is not yet able to parse past dates')\n    exch_map = {\n        'JP-HKD':['JP-TH'],\n        'JP-TH':['JP-TK'],\n        'JP-TK':['JP-TH', 'JP-CB'],\n        'JP-CB':['JP-TK', 'JP-HR', 'JP-KN'],\n        'JP-HR':['JP-CB', 'JP-KN'],\n        'JP-KN':['JP-CB', 'JP-HR', 'JP-SK', 'JP-CG'],\n        'JP-SK':['JP-KN', 'JP-CG'],\n        'JP-CG':['JP-KN', 'JP-SK', 'JP-KY']\n        }\n    df = fetch_consumption_df(zone_key, target_datetime)\n    df['imports'] = 0\n    for zone in exch_map[zone_key]:\n        df2 = occtonet.fetch_exchange(zone_key, zone, target_datetime)\n        df2 = pd.DataFrame(df2)\n        exchname = df2.loc[0, 'sortedZoneKeys']\n        df2 = df2[['datetime', 'netFlow']]\n        df2.columns = ['datetime', exchname]\n        df = pd.merge(df, df2, how='inner', on='datetime')\n        if exchname.split('->')[-1] == zone_key:\n            df['imports'] = df['imports']+df[exchname]\n        else:\n            df['imports'] = df['imports']-df[exchname]\n    df['prod'] = df['cons']-df['imports']\n    df = df[['datetime', 'prod']]\n    # add a row to production for each entry in the dictionary:\n    sources = {\n        'JP-HKD':'denkiyoho.hepco.co.jp',\n        'JP-TH':'setsuden.tohoku-epco.co.jp',\n        'JP-TK':'www.tepco.co.jp',\n        'JP-CB':'denki-yoho.chuden.jp',\n        'JP-HR':'www.rikuden.co.jp/denki-yoho',\n        'JP-KN':'www.kepco.co.jp',\n        'JP-SK':'www.energia.co.jp',\n        'JP-CG':'www.yonden.co.jp'\n        }\n    datalist = []\n    for i in range(df.shape[0]):\n        data = {\n            'zoneKey': zone_key,\n            'datetime': df.loc[i, 'datetime'].to_pydatetime(),\n            'production': {\n                'biomass': None,\n                'coal': None,\n                'gas': None,\n                'hydro': None,\n                'nuclear': None,\n                'oil': None,\n                'solar': None,\n                'wind': None,\n                'geothermal': None,\n                'unknown': df.loc[i, 'prod']\n            },\n            'storage': {},\n            'source': ['occtonet.or.jp', sources[zone_key]]\n            }\n        datalist.append(data)\n    return datalist\n\n\ndef fetch_consumption_df(zone_key='JP-TK', target_datetime=None,\n                      logger=logging.getLogger(__name__)):\n    \"\"\"\n    Returns the consumption for an area as a pandas DataFrame\n    \"\"\"\n    datestamp = arrow.get(target_datetime).to('Asia/Tokyo').strftime('%Y%m%d')\n    consumption_url = {\n        'JP-HKD': 'http://denkiyoho.hepco.co.jp/area/data/juyo_01_{}.csv'.format(datestamp),\n        'JP-TH': 'http://setsuden.tohoku-epco.co.jp/common/demand/juyo_02_{}.csv'.format(datestamp),\n        'JP-TK': 'http://www.tepco.co.jp/forecast/html/images/juyo-j.csv',\n        'JP-HR': 'http://www.rikuden.co.jp/denki-yoho/csv/juyo_05_{}.csv'.format(datestamp),\n        'JP-CB': 'http://denki-yoho.chuden.jp/denki_yoho_content_data/juyo_cepco003.csv',\n        'JP-KN': 'http://www.kepco.co.jp/yamasou/juyo1_kansai.csv',\n        'JP-CG': 'http://www.energia.co.jp/jukyuu/sys/juyo_07_{}.csv'.format(datestamp),\n        'JP-SK': 'http://www.yonden.co.jp/denkiyoho/juyo_shikoku.csv'\n        }\n    # First roughly 40 rows of the consumption files have hourly data,\n    # the parser skips to the rows with 5-min actual values \n    if zone_key == 'JP-KN':\n        startrow = 44\n    else:\n        startrow = 42\n    df = pd.read_csv(consumption_url[zone_key], skiprows=list(range(startrow)),\n                     encoding='shift-jis')\n    df.columns = ['Date', 'Time', 'cons']\n    # Convert \u4e07kW to MW\n    df['cons'] = 10*df['cons']\n    df = df.dropna()\n    df['datetime'] = df.apply(parse_dt, axis=1)\n    df = df[['datetime', 'cons']]\n    return df\n\n\ndef fetch_price(zone_key='JP-TK', session=None, target_datetime=None,\n                logger=logging.getLogger(__name__)):\n    if target_datetime is None:\n        target_datetime = dt.datetime.now() + dt.timedelta(days=1)\n\n    # price files contain data for fiscal year and not calendar year.\n    if target_datetime.month <= 3:\n        fiscal_year = target_datetime.year - 1\n    else:\n        fiscal_year = target_datetime.year\n    url = 'http://www.jepx.org/market/excel/spot_{}.csv'.format(fiscal_year)\n    df = pd.read_csv(url)\n\n    df = df.iloc[:, [0, 1, 6, 7, 8, 9, 10, 11, 12, 13, 14]]\n    df.columns = ['Date', 'Period', 'JP-HKD', 'JP-TH', 'JP-TK', 'JP-CB',\n                  'JP-HR', 'JP-KN', 'JP-CG', 'JP-SK', 'JP-KY']\n\n    if zone_key not in df.columns[2:]:\n        return []\n\n    start = target_datetime - dt.timedelta(days=1)\n    df['Date'] = df['Date'].apply(lambda x: dt.datetime.strptime(x, '%Y/%m/%d'))\n    df = df[(df['Date'] >= start.date()) & (df['Date'] <= target_datetime.date())]\n\n    df['datetime'] = df.apply(lambda row: arrow.get(row['Date']).shift(\n        minutes=30 * (row['Period'] - 1)).replace(tzinfo='Asia/Tokyo'), axis=1)\n\n    data = list()\n    for row in df.iterrows():\n        data.append({\n            'zoneKey': zone_key,\n            'currency': 'JPY',\n            'datetime': row[1]['datetime'].datetime,\n            'price': row[1][zone_key],\n            'source': 'jepx.org'\n        })\n\n    return data\n\n\ndef parse_dt(row):\n    \"\"\"\n    Parses timestamps from date and time\n    \"\"\"\n    return arrow.get(' '.join([row['Date'], row['Time']]).replace('/', '-'),\n                     'YYYY-M-D H:mm').replace(tzinfo='Asia/Tokyo').datetime\n\n\nif __name__ == '__main__':\n    \"\"\"Main method, never used by the Electricity Map backend, but handy for testing.\"\"\"\n\n    print('fetch_production() ->')\n    print(fetch_production())\n    print('fetch_price() ->')\n    print(fetch_price())\n", "path": "parsers/JP.py"}]}