text: #!/usr/bin/env python
  # -*- coding: utf-8 -*-
  #
  # Yahoo! Finance market data downloader (+fix for Pandas Datareader)
  # https://github.com/ranaroussi/yfinance
  #
  # Copyright 2017-2019 Ran Aroussi
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  
  from __future__ import print_function
  
  import time as _time
  import datetime as _datetime
  import requests as _requests
  import pandas as _pd
  import numpy as _np
  
  try:
      from urllib.parse import quote as urlencode
  except ImportError:
      from urllib import quote as urlencode
  
  from . import utils
  
  # import json as _json
  # import re as _re
  # import sys as _sys
  
  from . import shared
  
  
  class TickerBase():
      def __init__(self, ticker):
          self.ticker = ticker.upper()
          self._history = None
          self._base_url = 'https://query1.finance.yahoo.com'
          self._scrape_url = 'https://finance.yahoo.com/quote'
  
          self._fundamentals = False
          self._info = None
          self._sustainability = None
          self._recommendations = None
          self._major_holders = None
          self._institutional_holders = None
          self._mutualfund_holders = None
          self._isin = None
  
          self._calendar = None
          self._expirations = {}
  
          self._earnings = {
              "yearly": utils.empty_df(),
              "quarterly": utils.empty_df()}
          self._financials = {
              "yearly": utils.empty_df(),
              "quarterly": utils.empty_df()}
          self._balancesheet = {
              "yearly": utils.empty_df(),
              "quarterly": utils.empty_df()}
          self._cashflow = {
              "yearly": utils.empty_df(),
              "quarterly": utils.empty_df()}
  
      def history(self, period="1mo", interval="1d",
                  start=None, end=None, prepost=False, actions=True,
                  auto_adjust=True, back_adjust=False,
                  proxy=None, rounding=False, tz=None, **kwargs):
          """
          :Parameters:
              period : str
                  Valid periods: 1d,5d,1mo,3mo,6mo,1y,2y,5y,10y,ytd,max
                  Either Use period parameter or use start and end
              interval : str
                  Valid intervals: 1m,2m,5m,15m,30m,60m,90m,1h,1d,5d,1wk,1mo,3mo
                  Intraday data cannot extend last 60 days
              start: str
                  Download start date string (YYYY-MM-DD) or _datetime.
                  Default is 1900-01-01
              end: str
                  Download end date string (YYYY-MM-DD) or _datetime.
                  Default is now
              prepost : bool
                  Include Pre and Post market data in results?
                  Default is False
              auto_adjust: bool
                  Adjust all OHLC automatically? Default is True
              back_adjust: bool
                  Back-adjusted data to mimic true historical prices
              proxy: str
                  Optional. Proxy server URL scheme. Default is None
              rounding: bool
                  Round values to 2 decimal places?
                  Optional. Default is False = precision suggested by Yahoo!
              tz: str
                  Optional timezone locale for dates.
                  (default data is returned as non-localized dates)
              **kwargs: dict
                  debug: bool
                      Optional. If passed as False, will suppress
                      error message printing to console.
          """
  
          if start or period is None or period.lower() == "max":
              if start is None:
                  start = -2208988800
              elif isinstance(start, _datetime.datetime):
                  start = int(_time.mktime(start.timetuple()))
              else:
                  start = int(_time.mktime(
                      _time.strptime(str(start), '%Y-%m-%d')))
              if end is None:
                  end = int(_time.time())
              elif isinstance(end, _datetime.datetime):
                  end = int(_time.mktime(end.timetuple()))
              else:
                  end = int(_time.mktime(_time.strptime(str(end), '%Y-%m-%d')))
  
              params = {"period1": start, "period2": end}
          else:
              period = period.lower()
              params = {"range": period}
  
          params["interval"] = interval.lower()
          params["includePrePost"] = prepost
          params["events"] = "div,splits"
  
          # 1) fix weired bug with Yahoo! - returning 60m for 30m bars
          if params["interval"] == "30m":
              params["interval"] = "15m"
  
          # setup proxy in requests format
          if proxy is not None:
              if isinstance(proxy, dict) and "https" in proxy:
                  proxy = proxy["https"]
              proxy = {"https": proxy}
  
          # Getting data from json
          url = "{}/v8/finance/chart/{}".format(self._base_url, self.ticker)
          data = _requests.get(url=url, params=params, proxies=proxy)
          if "Will be right back" in data.text:
              raise RuntimeError("*** YAHOO! FINANCE IS CURRENTLY DOWN! ***\n"
                                 "Our engineers are working quickly to resolve "
                                 "the issue. Thank you for your patience.")
          data = data.json()
  
          # Work with errors
          debug_mode = True
          if "debug" in kwargs and isinstance(kwargs["debug"], bool):
              debug_mode = kwargs["debug"]
  
          err_msg = "No data found for this date range, symbol may be delisted"
          if "chart" in data and data["chart"]["error"]:
              err_msg = data["chart"]["error"]["description"]
              shared._DFS[self.ticker] = utils.empty_df()
              shared._ERRORS[self.ticker] = err_msg
              if "many" not in kwargs and debug_mode:
                  print('- %s: %s' % (self.ticker, err_msg))
              return shared._DFS[self.ticker]
  
          elif "chart" not in data or data["chart"]["result"] is None or \
                  not data["chart"]["result"]:
              shared._DFS[self.ticker] = utils.empty_df()
              shared._ERRORS[self.ticker] = err_msg
              if "many" not in kwargs and debug_mode:
                  print('- %s: %s' % (self.ticker, err_msg))
              return shared._DFS[self.ticker]
  
          # parse quotes
          try:
              quotes = utils.parse_quotes(data["chart"]["result"][0], tz)
          except Exception:
              shared._DFS[self.ticker] = utils.empty_df()
              shared._ERRORS[self.ticker] = err_msg
              if "many" not in kwargs and debug_mode:
                  print('- %s: %s' % (self.ticker, err_msg))
              return shared._DFS[self.ticker]
  
          # 2) fix weired bug with Yahoo! - returning 60m for 30m bars
          if interval.lower() == "30m":
              quotes2 = quotes.resample('30T')
              quotes = _pd.DataFrame(index=quotes2.last().index, data={
                  'Open': quotes2['Open'].first(),
                  'High': quotes2['High'].max(),
                  'Low': quotes2['Low'].min(),
                  'Close': quotes2['Close'].last(),
                  'Adj Close': quotes2['Adj Close'].last(),
                  'Volume': quotes2['Volume'].sum()
              })
              try:
                  quotes['Dividends'] = quotes2['Dividends'].max()
              except Exception:
                  pass
              try:
                  quotes['Stock Splits'] = quotes2['Dividends'].max()
              except Exception:
                  pass
  
          if auto_adjust:
              quotes = utils.auto_adjust(quotes)
          elif back_adjust:
              quotes = utils.back_adjust(quotes)
  
          if rounding:
              quotes = _np.round(quotes, data[
                  "chart"]["result"][0]["meta"]["priceHint"])
          quotes['Volume'] = quotes['Volume'].fillna(0).astype(_np.int64)
  
          quotes.dropna(inplace=True)
  
          # actions
          dividends, splits = utils.parse_actions(data["chart"]["result"][0], tz)
  
          # combine
          df = _pd.concat([quotes, dividends, splits], axis=1, sort=True)
          df["Dividends"].fillna(0, inplace=True)
          df["Stock Splits"].fillna(0, inplace=True)
  
          # index eod/intraday
          df.index = df.index.tz_localize("UTC").tz_convert(
              data["chart"]["result"][0]["meta"]["exchangeTimezoneName"])
  
          if params["interval"][-1] == "m":
              df.index.name = "Datetime"
          else:
              df.index = _pd.to_datetime(df.index.date)
              if tz is not None:
                  df.index = df.index.tz_localize(tz)
              df.index.name = "Date"
  
          self._history = df.copy()
  
          if not actions:
              df.drop(columns=["Dividends", "Stock Splits"], inplace=True)
  
          return df
  
      # ------------------------
  
      def _get_fundamentals(self, kind=None, proxy=None):
          def cleanup(data):
              df = _pd.DataFrame(data).drop(columns=['maxAge'])
              for col in df.columns:
                  df[col] = _np.where(
                      df[col].astype(str) == '-', _np.nan, df[col])
  
              df.set_index('endDate', inplace=True)
              try:
                  df.index = _pd.to_datetime(df.index, unit='s')
              except ValueError:
                  df.index = _pd.to_datetime(df.index)
              df = df.T
              df.columns.name = ''
              df.index.name = 'Breakdown'
  
              df.index = utils.camel2title(df.index)
              return df
  
          # setup proxy in requests format
          if proxy is not None:
              if isinstance(proxy, dict) and "https" in proxy:
                  proxy = proxy["https"]
              proxy = {"https": proxy}
  
          if self._fundamentals:
              return
  
          # get info and sustainability
          url = '%s/%s' % (self._scrape_url, self.ticker)
          data = utils.get_json(url, proxy)
  
          # holders
          #url = "{}/{}/holders".format(self._scrape_url, self.ticker)
          holders = _pd.read_html(url+'/holders')
          
          if len(holders)>=3:
              self._major_holders = holders[0]
              self._institutional_holders = holders[1]
              self._mutualfund_holders = holders[2]
          elif len(holders)>=2:
              self._major_holders = holders[0]
              self._institutional_holders = holders[1]
          else:
              self._major_holders = holders[0]
          
          #self._major_holders = holders[0]
          #self._institutional_holders = holders[1]
          
          if self._institutional_holders is not None:
              if 'Date Reported' in self._institutional_holders:
                  self._institutional_holders['Date Reported'] = _pd.to_datetime(
                  self._institutional_holders['Date Reported'])
              if '% Out' in self._institutional_holders:
                  self._institutional_holders['% Out'] = self._institutional_holders[
                  '% Out'].str.replace('%', '').astype(float)/100
  
          if self._mutualfund_holders is not None:
              if 'Date Reported' in self._mutualfund_holders:
                  self._mutualfund_holders['Date Reported'] = _pd.to_datetime(
                  self._mutualfund_holders['Date Reported'])
              if '% Out' in self._mutualfund_holders:
                  self._mutualfund_holders['% Out'] = self._mutualfund_holders[
                  '% Out'].str.replace('%', '').astype(float)/100
  
          # sustainability
          d = {}
          if isinstance(data.get('esgScores'), dict):
              for item in data['esgScores']:
                  if not isinstance(data['esgScores'][item], (dict, list)):
                      d[item] = data['esgScores'][item]
  
              s = _pd.DataFrame(index=[0], data=d)[-1:].T
              s.columns = ['Value']
              s.index.name = '%.f-%.f' % (
                  s[s.index == 'ratingYear']['Value'].values[0],
                  s[s.index == 'ratingMonth']['Value'].values[0])
  
              self._sustainability = s[~s.index.isin(
                  ['maxAge', 'ratingYear', 'ratingMonth'])]
  
          # info (be nice to python 2)
          self._info = {}
          items = ['summaryProfile', 'summaryDetail', 'quoteType',
                   'defaultKeyStatistics', 'assetProfile', 'summaryDetail']
          for item in items:
              if isinstance(data.get(item), dict):
                  self._info.update(data[item])
  
          self._info['regularMarketPrice'] = self._info['regularMarketOpen']
          self._info['logo_url'] = ""
          try:
              domain = self._info['website'].split(
                  '://')[1].split('/')[0].replace('www.', '')
              self._info['logo_url'] = 'https://logo.clearbit.com/%s' % domain
          except Exception:
              pass
  
          # events
          try:
              cal = _pd.DataFrame(
                  data['calendarEvents']['earnings'])
              cal['earningsDate'] = _pd.to_datetime(
                  cal['earningsDate'], unit='s')
              self._calendar = cal.T
              self._calendar.index = utils.camel2title(self._calendar.index)
              self._calendar.columns = ['Value']
          except Exception:
              pass
  
          # analyst recommendations
          try:
              rec = _pd.DataFrame(
                  data['upgradeDowngradeHistory']['history'])
              rec['earningsDate'] = _pd.to_datetime(
                  rec['epochGradeDate'], unit='s')
              rec.set_index('earningsDate', inplace=True)
              rec.index.name = 'Date'
              rec.columns = utils.camel2title(rec.columns)
              self._recommendations = rec[[
                  'Firm', 'To Grade', 'From Grade', 'Action']].sort_index()
          except Exception:
              pass
  
          # get fundamentals
          data = utils.get_json(url+'/financials', proxy)
  
          # generic patterns
          for key in (
              (self._cashflow, 'cashflowStatement', 'cashflowStatements'),
              (self._balancesheet, 'balanceSheet', 'balanceSheetStatements'),
              (self._financials, 'incomeStatement', 'incomeStatementHistory')
          ):
  
              item = key[1] + 'History'
              if isinstance(data.get(item), dict):
                  key[0]['yearly'] = cleanup(data[item][key[2]])
  
              item = key[1]+'HistoryQuarterly'
              if isinstance(data.get(item), dict):
                  key[0]['quarterly'] = cleanup(data[item][key[2]])
  
          # earnings
          if isinstance(data.get('earnings'), dict):
              earnings = data['earnings']['financialsChart']
              df = _pd.DataFrame(earnings['yearly']).set_index('date')
              df.columns = utils.camel2title(df.columns)
              df.index.name = 'Year'
              self._earnings['yearly'] = df
  
              df = _pd.DataFrame(earnings['quarterly']).set_index('date')
              df.columns = utils.camel2title(df.columns)
              df.index.name = 'Quarter'
              self._earnings['quarterly'] = df
  
          self._fundamentals = True
  
      def get_recommendations(self, proxy=None, as_dict=False, *args, **kwargs):
          self._get_fundamentals(proxy)
          data = self._recommendations
          if as_dict:
              return data.to_dict()
          return data
  
      def get_calendar(self, proxy=None, as_dict=False, *args, **kwargs):
          self._get_fundamentals(proxy)
          data = self._calendar
          if as_dict:
              return data.to_dict()
          return data
  
      def get_major_holders(self, proxy=None, as_dict=False, *args, **kwargs):
          self._get_fundamentals(proxy)
          data = self._major_holders
          if as_dict:
              return data.to_dict()
          return data
  
      def get_institutional_holders(self, proxy=None, as_dict=False, *args, **kwargs):
          self._get_fundamentals(proxy)
          data = self._institutional_holders
          if data is not None:
              if as_dict:
                  return data.to_dict()
              return data
  
      def get_mutualfund_holders(self, proxy=None, as_dict=False, *args, **kwargs):
          self._get_fundamentals(proxy)
          data = self._mutualfund_holders
          if data is not None:
              if as_dict:
                  return data.to_dict()
              return data
  
      def get_info(self, proxy=None, as_dict=False, *args, **kwargs):
          self._get_fundamentals(proxy)
          data = self._info
          if as_dict:
              return data.to_dict()
          return data
  
      def get_sustainability(self, proxy=None, as_dict=False, *args, **kwargs):
          self._get_fundamentals(proxy)
          data = self._sustainability
          if as_dict:
              return data.to_dict()
          return data
  
      def get_earnings(self, proxy=None, as_dict=False, freq="yearly"):
          self._get_fundamentals(proxy)
          data = self._earnings[freq]
          if as_dict:
              return data.to_dict()
          return data
  
      def get_financials(self, proxy=None, as_dict=False, freq="yearly"):
          self._get_fundamentals(proxy)
          data = self._financials[freq]
          if as_dict:
              return data.to_dict()
          return data
  
      def get_balancesheet(self, proxy=None, as_dict=False, freq="yearly"):
          self._get_fundamentals(proxy)
          data = self._balancesheet[freq]
          if as_dict:
              return data.to_dict()
          return data
  
      def get_balance_sheet(self, proxy=None, as_dict=False, freq="yearly"):
          return self.get_balancesheet(proxy, as_dict, freq)
  
      def get_cashflow(self, proxy=None, as_dict=False, freq="yearly"):
          self._get_fundamentals(proxy)
          data = self._cashflow[freq]
          if as_dict:
              return data.to_dict()
          return data
  
      def get_dividends(self, proxy=None):
          if self._history is None:
              self.history(period="max", proxy=proxy)
          dividends = self._history["Dividends"]
          return dividends[dividends != 0]
  
      def get_splits(self, proxy=None):
          if self._history is None:
              self.history(period="max", proxy=proxy)
          splits = self._history["Stock Splits"]
          return splits[splits != 0]
  
      def get_actions(self, proxy=None):
          if self._history is None:
              self.history(period="max", proxy=proxy)
          actions = self._history[["Dividends", "Stock Splits"]]
          return actions[actions != 0].dropna(how='all').fillna(0)
  
      def get_isin(self, proxy=None):
          # *** experimental ***
          if self._isin is not None:
              return self._isin
  
          ticker = self.ticker.upper()
  
          if "-" in ticker or "^" in ticker:
              self._isin = '-'
              return self._isin
  
          # setup proxy in requests format
          if proxy is not None:
              if isinstance(proxy, dict) and "https" in proxy:
                  proxy = proxy["https"]
              proxy = {"https": proxy}
  
          q = ticker
          self.get_info(proxy=proxy)
          if "shortName" in self._info:
              q = self._info['shortName']
  
          url = 'https://markets.businessinsider.com/ajax/' \
                'SearchController_Suggest?max_results=25&query=%s' \
              % urlencode(q)
          data = _requests.get(url=url, proxies=proxy).text
  
          search_str = '"{}|'.format(ticker)
          if search_str not in data:
              if q.lower() in data.lower():
                  search_str = '"|'
                  if search_str not in data:
                      self._isin = '-'
                      return self._isin
              else:
                  self._isin = '-'
                  return self._isin
  
          self._isin = data.split(search_str)[1].split('"')[0].split('|')[0]
          return self._isin
  