text: #!/usr/local/bin/python 
  
  #GENERATED BY /aiservices/notebooks/stocks/01_GetData.ipynb
  
  import os, datetime, glob, sys, time, datetime, colabexts, shutil, argparse, csv, requests
  import matplotlib as mpl
  import matplotlib.pyplot as plt
  import numpy as np
  import pandas as pd
  from colabexts.jcommon import *
  from alpha_vantage.timeseries import TimeSeries
  from alpha_vantage.fundamentaldata import FundamentalData
  import yfinance as yf
  
  sys.path.append("/opt/data/data/stocks/")
  from utils import *
  
  
  API_KEY = None  # Put your API KEY if you need to test data download or just use the data
  API_KEY, NEWS_KEY = getkey()
  
  #-----------------------------------------------------------------------------------
  '''
  This will read required symbols and saves them to data directory
  '''
  def chk_recent(filename, elapsed=8*60*60 ):
      if (os.path.exists(filename)):
          dt   = datetime.datetime.fromtimestamp(os.path.getmtime(filename))
          dn   = datetime.datetime.now()
          ts   = (dn - dt)
          secs = (ts.days * 24 * 60 * 60 + ts.seconds)
          if (secs < elapsed): 
              print(f"{filename:22} exists, ... crested less than {secs} seconds!! at {dt} ")
              return True;
          
      return False
  
  #-----------------------------------------------------------------------------------
  def get_intraday_extended(symbol='msft', apikey=API_KEY):
      CSV_URL = 'https://www.alphavantage.co/query?function=TIME_SERIES_INTRADAY_EXTENDED&interval=15min&slice=year1month1&'
      CSV_URL += f'symbol={symbol}&apikey={apikey}'
      
      df = None;
      with requests.Session() as s:
          download = s.get(CSV_URL)
          decoded_content = download.content.decode('utf-8')
          cr = csv.reader(decoded_content.splitlines(), delimiter=',')
          my_list = list(cr)
          df = pd.DataFrame(my_list[1:])
          df.columns = my_list[0]
          for i, row in enumerate(my_list):
              print(row)
              if (i > 10):
                  break;
      return df
  
  #-----------------------------------------------------------------------------------
  def get_fundamentals(symbol='msft', API_KEY=API_KEY, savein="/opt/data/data/stocks/data/"):
      if (savein and not savein.endswith("/")):
          savein = savein + "/"
          
      symbol = symbol.lower()
      outf = f'{savein}/{symbol}/fundamentals.json'
      
      # If no fundamentals data is available for ex: SPY then dont get them
      if ( os.path.exists(f'{outf}.none')):
          return 0
      
      if chk_recent (outf, 24*60*60):  # Get this once a day or two should be good
          return 0;
      
      data = None
      try:
          fd = FundamentalData(key=API_KEY)
          data = fd.get_company_overview(symbol)
          print( f"saving fundamentals to output: {outf}" )
          with( open(outf, "w+")) as f:
              f.write(json.dumps(data[0], indent=3) )
          return data[0]
      except Exception as e: 
          print(f"ERROR: in get_fundamentals for: {symbol} {e}")
          return None
      
  #-----------------------------------------------------------------------------------
  '''
  This will read required symbols and saves them to data directory
  '''
  def save_daily(symbol, API_KEY=API_KEY, savein="/opt/data/data/stocks/data/"):
      if (savein and not savein.endswith("/")):
          savein = savein + "/"
          
      symbol = symbol.lower()
      outf = f'{savein}/{symbol}/daily.csv'
      if chk_recent (outf, 4*60*60):
          return 0;
      
      ts = TimeSeries(key=API_KEY, output_format='pandas')
      
      data = None
      try:
          print(f"Getting data for {symbol}")
          data ,  meta_data = ts.get_daily   (symbol, outputsize='full')
      except ValueError as ve:
          print(f"Exception in (save_daily) for {symbol} {ve}");
              
      if data is None:
          print(f"ERROR save_daily: Could not get data for {symbol}")
          return data
      
      data.insert(0, 'timestamp', value=data.index)
  
      data.columns = 'timestamp,open,high,low,close,volume'.split(',')
      print( f"saving to output: {outf}" )
      data.to_csv(outf, index=False)
      return data
  
  #-----------------------------------------------------------------------------------
  '''
  This will read required symbols and saves them to data directory
  '''
  def save_intraday(symbol, API_KEY=API_KEY, check=True, savein="/opt/data/data/stocks/data/"):
      from alpha_vantage.timeseries import TimeSeries
      if (savein and not savein.endswith("/")):
          savein = savein + "/"
          
      symbol = symbol.lower()
      outf = f'{savein}/{symbol}/daily_ext.csv'
      if chk_recent (outf, 4*60*60):
          return 0;
      
      ts = TimeSeries(key=API_KEY, output_format='pandas')
      
      data = None
      try:
          print(f"Getting extended data for {symbol}")
          data ,  meta_data = ts.get_intraday(symbol, outputsize='full')
      except ValueError as ve:
          print(f"save_daily_ext ... {ve}");
              
      if data is None:
          print(f"ERROR: save_intraday - Could not get data for {symbol}")
          return None
      
      data.insert(0, 'timestamp', value=data.index)
  
      data.columns = 'timestamp,open,high,low,close,volume'.split(',')
      print( f"saving to output: {outf}" )
      data.to_csv(outf, index=False)
      return data
  
  #-----------------------------------------------------------------------------------
  '''
  This will read required symbols and saves them to data directory
  '''
  def save_intraYahoo(symbol, savein="/opt/data/data/stocks/data/"):
      symbol = symbol.lower()
      outf = f'{savein}/{symbol}/daily_intraYahoo.csv'
      if chk_recent (outf, 15*60):
          return 0;
      
      data = yf.download(tickers=symbol, period='1d', interval='1m')
      data.drop(['Adj Close'], errors='ignore', inplace=True, axis=1)
      data.insert(0, 'timestamp', value=data.index)
      
      print( f"saving to output: {outf}" )
      data.to_csv(outf, index=False)
  
      return data;
  #-----------------------------------------------------------------------------------
  def getdata(symbols='MSFT GLD GOOGL SPX AAPL IBM' ):
      global API_KEY  
      API_KEY, NEWS_API_KEY = API_KEY, None or getkey()
  
      errors = [];
      for f in symbols.split():
          if ( not f or f.startswith ('#')):
              print( f'ignoring {f}')
              continue; # empty lines
          kv = f.split(':')
          k = kv[0]
          v = kv[1] if len(kv) > 1 else k 
          print(f'---------------------------------\n+Getting data for {k} => symbol {v}')
          
          ret = save_daily(k, API_KEY=API_KEY)
          if (ret is None ): errors.append(f + ": save_daily")
          ret = save_intraday (k, API_KEY=API_KEY)
          if (ret is None ): errors.append(f + ": save_intraday")
          ret = save_intraYahoo (k)
          if (ret is None ): errors.append(f +": save_intraYahoo")
          ret = get_fundamentals(k)
          if (ret is None ): errors.append(f + ": get_fundamentals")
          
      if len(errors) > 0:
          print ("ERRORS OCCURED: \n\n", errors)
          
  #-----------------------------------------------------------------------------------
  def main(path="/opt/data/data/stocks/data/"):
      for (dirpath, dirnames, filenames) in os.walk(path):
          for d in dirnames:
              if ( os.path.exists(f'{path}{d}/daily.csv')):
                  #print(f"+ Trying {d} getdata({d})")
                  getdata(d)
                  pass
  #-----------------------------------------------------------------------------------
  sysargs = None
  ASIA    = '6758.T:TOKYO  6501.T:HITACHI 0168.HK:HNGKNG 601288.SS:SHANGAI  SHNZEN'
  ASIA    = '601288.SS:SHANGAI'
  defsyms = f"MSFT GLD GOOGL AAPL IBM {ASIA}"
  defsyms = f""
  deffile = "symbols.txt"
  defoutp = "data/stock_price/stockdata.csv"
  def addargs():
      global sysargs
      p = argparse.ArgumentParser(f"{os.path.basename(sys.argv[0])}:")
      p.add_argument('-s', '--symbols', type=str, default=defsyms, help=f"Symbols separated ex: '{defsyms}'")
      #p.add_argument('-f', '--file'   , type=str, default=deffile, help=f"Symbols from file ex: '{deffile}'")
      #p.add_argument('-o', '--output' , type=str, default=defoutp, help= f"o/p def: {defoutp}")
      p.add_argument('-r', '--runall' , action='store_false', help= f"DO NOT RUN in a directory")
  #    p.add_argument('args', nargs=argparse.REMAINDER)
  #    p.add_argument('input_files',action="store", type=str, nargs='+', help="input file(s)")
  
      try:
          sysargs, unknown=p.parse_known_args(sys.argv[1:])
      except argparse.ArgumentError as exc:
          print(exc.message )
          
      if (unknown):
          print("Unknown options: ", unknown)
          #p.print_help()
      return sysargs
  #-----------------------------------------------------------------------------------
  if __name__ == '__main__':
      if (not inJupyter()):
          t1      = datetime.datetime.now()
          sysargs = addargs()
          if ( sysargs.runall ):
              main()
  
          symbs   = sysargs.symbols
          if (symbs):
              getdata(symbs)
              
          t2 = datetime.datetime.now()
          print(f"All Done in {str(t2-t1)} ***")
      else:
          pass
          '''
          a, minrows, ldf  = read_data()
          stockfile="data/stockdata.csv"
          af= combine_data(a, stockfile)
          '''
  