text: #
  # Copyright 2016 Quantopian, Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  import os
  from collections import OrderedDict
  
  import logbook
  import pandas as pd
  from pandas_datareader.data import DataReader
  import pytz
  from six import iteritems
  from six.moves.urllib_error import HTTPError
  
  from .benchmarks import get_benchmark_returns
  from . import treasuries, treasuries_can
  from ..utils.paths import (
      cache_root,
      data_root,
  )
  from ..utils.deprecate import deprecated
  from zipline.utils.calendars import get_calendar
  
  logger = logbook.Logger('Loader')
  
  # Mapping from index symbol to appropriate bond data
  INDEX_MAPPING = {
      '^GSPC':
      (treasuries, 'treasury_curves.csv', 'www.federalreserve.gov'),
      '^GSPTSE':
      (treasuries_can, 'treasury_curves_can.csv', 'bankofcanada.ca'),
      '^FTSE':  # use US treasuries until UK bonds implemented
      (treasuries, 'treasury_curves.csv', 'www.federalreserve.gov'),
  }
  
  ONE_HOUR = pd.Timedelta(hours=1)
  
  nyse_cal = get_calendar('NYSE')
  trading_day_nyse = nyse_cal.day
  trading_days_nyse = nyse_cal.all_sessions
  
  
  def last_modified_time(path):
      """
      Get the last modified time of path as a Timestamp.
      """
      return pd.Timestamp(os.path.getmtime(path), unit='s', tz='UTC')
  
  
  def get_data_filepath(name):
      """
      Returns a handle to data file.
  
      Creates containing directory, if needed.
      """
      dr = data_root()
  
      if not os.path.exists(dr):
          os.makedirs(dr)
  
      return os.path.join(dr, name)
  
  
  def get_cache_filepath(name):
      cr = cache_root()
      if not os.path.exists(cr):
          os.makedirs(cr)
  
      return os.path.join(cr, name)
  
  
  def get_benchmark_filename(symbol):
      return "%s_benchmark.csv" % symbol
  
  
  def has_data_for_dates(series_or_df, first_date, last_date):
      """
      Does `series_or_df` have data on or before first_date and on or after
      last_date?
      """
      dts = series_or_df.index
      if not isinstance(dts, pd.DatetimeIndex):
          raise TypeError("Expected a DatetimeIndex, but got %s." % type(dts))
      first, last = dts[[0, -1]]
      return (first <= first_date) and (last >= last_date)
  
  
  def load_market_data(trading_day=trading_day_nyse,
                       trading_days=trading_days_nyse,
                       bm_symbol='^GSPC'):
      """
      Load benchmark returns and treasury yield curves for the given calendar and
      benchmark symbol.
  
      Benchmarks are downloaded as a Series from Yahoo Finance.  Treasury curves
      are US Treasury Bond rates and are downloaded from 'www.federalreserve.gov'
      by default.  For Canadian exchanges, a loader for Canadian bonds from the
      Bank of Canada is also available.
  
      Results downloaded from the internet are cached in
      ~/.zipline/data. Subsequent loads will attempt to read from the cached
      files before falling back to redownload.
  
      Parameters
      ----------
      trading_day : pandas.CustomBusinessDay, optional
          A trading_day used to determine the latest day for which we
          expect to have data.  Defaults to an NYSE trading day.
      trading_days : pd.DatetimeIndex, optional
          A calendar of trading days.  Also used for determining what cached
          dates we should expect to have cached. Defaults to the NYSE calendar.
      bm_symbol : str, optional
          Symbol for the benchmark index to load.  Defaults to '^GSPC', the Yahoo
          ticker for the S&P 500.
  
      Returns
      -------
      (benchmark_returns, treasury_curves) : (pd.Series, pd.DataFrame)
  
      Notes
      -----
  
      Both return values are DatetimeIndexed with values dated to midnight in UTC
      of each stored date.  The columns of `treasury_curves` are:
  
      '1month', '3month', '6month',
      '1year','2year','3year','5year','7year','10year','20year','30year'
      """
      first_date = trading_days[0]
      now = pd.Timestamp.utcnow()
  
      # We expect to have benchmark and treasury data that's current up until
      # **two** full trading days prior to the most recently completed trading
      # day.
      # Example:
      # On Thu Oct 22 2015, the previous completed trading day is Wed Oct 21.
      # However, data for Oct 21 doesn't become available until the early morning
      # hours of Oct 22.  This means that there are times on the 22nd at which we
      # cannot reasonably expect to have data for the 21st available.  To be
      # conservative, we instead expect that at any time on the 22nd, we can
      # download data for Tuesday the 20th, which is two full trading days prior
      # to the date on which we're running a test.
  
      # We'll attempt to download new data if the latest entry in our cache is
      # before this date.
      last_date = trading_days[trading_days.get_loc(now, method='ffill') - 2]
  
      br = ensure_benchmark_data(
          bm_symbol,
          first_date,
          last_date,
          now,
          # We need the trading_day to figure out the close prior to the first
          # date so that we can compute returns for the first date.
          trading_day,
      )
      tc = ensure_treasury_data(
          bm_symbol,
          first_date,
          last_date,
          now,
      )
      benchmark_returns = br[br.index.slice_indexer(first_date, last_date)]
      treasury_curves = tc[tc.index.slice_indexer(first_date, last_date)]
      return benchmark_returns, treasury_curves
  
  
  def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day):
      """
      Ensure we have benchmark data for `symbol` from `first_date` to `last_date`
  
      Parameters
      ----------
      symbol : str
          The symbol for the benchmark to load.
      first_date : pd.Timestamp
          First required date for the cache.
      last_date : pd.Timestamp
          Last required date for the cache.
      now : pd.Timestamp
          The current time.  This is used to prevent repeated attempts to
          re-download data that isn't available due to scheduling quirks or other
          failures.
      trading_day : pd.CustomBusinessDay
          A trading day delta.  Used to find the day before first_date so we can
          get the close of the day prior to first_date.
  
      We attempt to download data unless we already have data stored at the data
      cache for `symbol` whose first entry is before or on `first_date` and whose
      last entry is on or after `last_date`.
  
      If we perform a download and the cache criteria are not satisfied, we wait
      at least one hour before attempting a redownload.  This is determined by
      comparing the current time to the result of os.path.getmtime on the cache
      path.
      """
      path = get_data_filepath(get_benchmark_filename(symbol))
  
      # If the path does not exist, it means the first download has not happened
      # yet, so don't try to read from 'path'.
      if os.path.exists(path):
          try:
              data = pd.Series.from_csv(path).tz_localize('UTC')
              if has_data_for_dates(data, first_date, last_date):
                  return data
  
              # Don't re-download if we've successfully downloaded and written a
              # file in the last hour.
              last_download_time = last_modified_time(path)
              if (now - last_download_time) <= ONE_HOUR:
                  logger.warn(
                      "Refusing to download new benchmark data because a "
                      "download succeeded at %s." % last_download_time
                  )
                  return data
  
          except (OSError, IOError, ValueError) as e:
              # These can all be raised by various versions of pandas on various
              # classes of malformed input.  Treat them all as cache misses.
              logger.info(
                  "Loading data for {path} failed with error [{error}].".format(
                      path=path, error=e,
                  )
              )
      logger.info(
          "Cache at {path} does not have data from {start} to {end}.\n"
          "Downloading benchmark data for '{symbol}'.",
          start=first_date,
          end=last_date,
          symbol=symbol,
          path=path,
      )
  
      try:
          data = get_benchmark_returns(
              symbol,
              first_date - trading_day,
              last_date,
          )
          data.to_csv(path)
      except (OSError, IOError, HTTPError):
          logger.exception('failed to cache the new benchmark returns')
      if not has_data_for_dates(data, first_date, last_date):
          logger.warn("Still don't have expected data after redownload!")
      return data
  
  
  def ensure_treasury_data(bm_symbol, first_date, last_date, now):
      """
      Ensure we have treasury data from treasury module associated with
      `bm_symbol`.
  
      Parameters
      ----------
      bm_symbol : str
          Benchmark symbol for which we're loading associated treasury curves.
      first_date : pd.Timestamp
          First date required to be in the cache.
      last_date : pd.Timestamp
          Last date required to be in the cache.
      now : pd.Timestamp
          The current time.  This is used to prevent repeated attempts to
          re-download data that isn't available due to scheduling quirks or other
          failures.
  
      We attempt to download data unless we already have data stored in the cache
      for `module_name` whose first entry is before or on `first_date` and whose
      last entry is on or after `last_date`.
  
      If we perform a download and the cache criteria are not satisfied, we wait
      at least one hour before attempting a redownload.  This is determined by
      comparing the current time to the result of os.path.getmtime on the cache
      path.
      """
      loader_module, filename, source = INDEX_MAPPING.get(
          bm_symbol, INDEX_MAPPING['^GSPC']
      )
      first_date = max(first_date, loader_module.earliest_possible_date())
      path = get_data_filepath(filename)
  
      # If the path does not exist, it means the first download has not happened
      # yet, so don't try to read from 'path'.
      if os.path.exists(path):
          try:
              data = pd.DataFrame.from_csv(path).tz_localize('UTC')
              if has_data_for_dates(data, first_date, last_date):
                  return data
  
              # Don't re-download if we've successfully downloaded and written a
              # file in the last hour.
              last_download_time = last_modified_time(path)
              if (now - last_download_time) <= ONE_HOUR:
                  logger.warn(
                      "Refusing to download new treasury data because a "
                      "download succeeded at %s." % last_download_time
                  )
                  return data
  
          except (OSError, IOError, ValueError) as e:
              # These can all be raised by various versions of pandas on various
              # classes of malformed input.  Treat them all as cache misses.
              logger.info(
                  "Loading data for {path} failed with error [{error}].".format(
                      path=path, error=e,
                  )
              )
  
      try:
          data = loader_module.get_treasury_data(first_date, last_date)
          data.to_csv(path)
      except (OSError, IOError, HTTPError):
          logger.exception('failed to cache treasury data')
      if not has_data_for_dates(data, first_date, last_date):
          logger.warn("Still don't have expected data after redownload!")
      return data
  
  
  def _load_raw_yahoo_data(indexes=None, stocks=None, start=None, end=None):
      """Load closing prices from yahoo finance.
  
      :Optional:
          indexes : dict (Default: {'SPX': '^GSPC'})
              Financial indexes to load.
          stocks : list (Default: ['AAPL', 'GE', 'IBM', 'MSFT',
                                   'XOM', 'AA', 'JNJ', 'PEP', 'KO'])
              Stock closing prices to load.
          start : datetime (Default: datetime(1993, 1, 1, 0, 0, 0, 0, pytz.utc))
              Retrieve prices from start date on.
          end : datetime (Default: datetime(2002, 1, 1, 0, 0, 0, 0, pytz.utc))
              Retrieve prices until end date.
  
      :Note:
          This is based on code presented in a talk by Wes McKinney:
          http://wesmckinney.com/files/20111017/notebook_output.pdf
      """
      assert indexes is not None or stocks is not None, """
  must specify stocks or indexes"""
  
      if start is None:
          start = pd.datetime(1990, 1, 1, 0, 0, 0, 0, pytz.utc)
  
      if start is not None and end is not None:
          assert start < end, "start date is later than end date."
  
      data = OrderedDict()
  
      if stocks is not None:
          for stock in stocks:
              logger.info('Loading stock: {}'.format(stock))
              stock_pathsafe = stock.replace(os.path.sep, '--')
              cache_filename = "{stock}-{start}-{end}.csv".format(
                  stock=stock_pathsafe,
                  start=start,
                  end=end).replace(':', '-')
              cache_filepath = get_cache_filepath(cache_filename)
              if os.path.exists(cache_filepath):
                  stkd = pd.DataFrame.from_csv(cache_filepath)
              else:
                  stkd = DataReader(stock, 'yahoo', start, end).sort_index()
                  stkd.to_csv(cache_filepath)
              data[stock] = stkd
  
      if indexes is not None:
          for name, ticker in iteritems(indexes):
              logger.info('Loading index: {} ({})'.format(name, ticker))
              stkd = DataReader(ticker, 'yahoo', start, end).sort_index()
              data[name] = stkd
  
      return data
  
  
  def load_from_yahoo(indexes=None,
                      stocks=None,
                      start=None,
                      end=None,
                      adjusted=True):
      """
      Loads price data from Yahoo into a dataframe for each of the indicated
      assets.  By default, 'price' is taken from Yahoo's 'Adjusted Close',
      which removes the impact of splits and dividends. If the argument
      'adjusted' is False, then the non-adjusted 'close' field is used instead.
  
      :param indexes: Financial indexes to load.
      :type indexes: dict
      :param stocks: Stock closing prices to load.
      :type stocks: list
      :param start: Retrieve prices from start date on.
      :type start: datetime
      :param end: Retrieve prices until end date.
      :type end: datetime
      :param adjusted: Adjust the price for splits and dividends.
      :type adjusted: bool
  
      """
      data = _load_raw_yahoo_data(indexes, stocks, start, end)
      if adjusted:
          close_key = 'Adj Close'
      else:
          close_key = 'Close'
      df = pd.DataFrame({key: d[close_key] for key, d in iteritems(data)})
      df.index = df.index.tz_localize(pytz.utc)
      return df
  
  
  @deprecated(
      'load_bars_from_yahoo is deprecated, please register a'
      ' yahoo_equities data bundle instead',
  )
  def load_bars_from_yahoo(indexes=None,
                           stocks=None,
                           start=None,
                           end=None,
                           adjusted=True):
      """
      Loads data from Yahoo into a panel with the following
      column names for each indicated security:
  
          - open
          - high
          - low
          - close
          - volume
          - price
  
      Note that 'price' is Yahoo's 'Adjusted Close', which removes the
      impact of splits and dividends. If the argument 'adjusted' is True, then
      the open, high, low, and close values are adjusted as well.
  
      :param indexes: Financial indexes to load.
      :type indexes: dict
      :param stocks: Stock closing prices to load.
      :type stocks: list
      :param start: Retrieve prices from start date on.
      :type start: datetime
      :param end: Retrieve prices until end date.
      :type end: datetime
      :param adjusted: Adjust open/high/low/close for splits and dividends.
          The 'price' field is always adjusted.
      :type adjusted: bool
  
      """
      data = _load_raw_yahoo_data(indexes, stocks, start, end)
      panel = pd.Panel(data)
      # Rename columns
      panel.minor_axis = ['open', 'high', 'low', 'close', 'volume', 'price']
      panel.major_axis = panel.major_axis.tz_localize(pytz.utc)
      # Adjust data
      if adjusted:
          adj_cols = ['open', 'high', 'low', 'close']
          for ticker in panel.items:
              ratio = (panel[ticker]['price'] / panel[ticker]['close'])
              ratio_filtered = ratio.fillna(0).values
              for col in adj_cols:
                  panel[ticker][col] *= ratio_filtered
      return panel
  
  
  def load_prices_from_csv(filepath, identifier_col, tz='UTC'):
      data = pd.read_csv(filepath, index_col=identifier_col)
      data.index = pd.DatetimeIndex(data.index, tz=tz)
      data.sort_index(inplace=True)
      return data
  
  
  def load_prices_from_csv_folder(folderpath, identifier_col, tz='UTC'):
      data = None
      for file in os.listdir(folderpath):
          if '.csv' not in file:
              continue
          raw = load_prices_from_csv(os.path.join(folderpath, file),
                                     identifier_col, tz)
          if data is None:
              data = raw
          else:
              data = pd.concat([data, raw], axis=1)
      return data
  