text: """Project: Eskapade - A python-based package for data analysis.
  
  Macro: esk303_hgr_filler_plotter
  
  Created: 2017/02/17
  
  Description:
      Macro that illustrates how to loop over multiple (possibly large!)
      datasets in chunks, in each loop fill a histogrammar histograms,
      and plot the final histograms.
  
  Authors:
      KPMG Advanced Analytics & Big Data team, Amstelveen, The Netherlands
  
  Redistribution and use in source and binary forms, with or without
  modification, are permitted according to the terms listed in the file
  LICENSE.
  """
  
  from eskapade import analysis, core_ops, process_manager, resources, visualization, ConfigObject, Chain
  from eskapade.logger import Logger, LogLevel
  
  logger = Logger()
  
  logger.debug('Now parsing configuration file esk303_hgr_filler_plotter.py.')
  
  #########################################################################################
  # --- minimal analysis information
  settings = process_manager.service(ConfigObject)
  settings['analysisName'] = 'esk303_hgr_filler_plotter'
  settings['version'] = 0
  
  #########################################################################################
  
  msg = r"""
  
  The plots and latex files produced by link hist_summary can be found in dir:
  {path}
  """
  logger.info(msg, path=settings['resultsDir'] + '/' + settings['analysisName'] + '/data/v0/report/')
  
  # --- Analysis configuration flags.
  #     E.g. use these flags turn on or off certain chains with links.
  #     by default all set to false, unless already configured in
  #     configobject or vars()
  
  settings['do_loop'] = True
  settings['do_plotting'] = True
  
  chunk_size = 400
  
  #########################################################################################
  # --- create dummy example dataset, which is read in below
  
  input_files = [resources.fixture('mock_accounts.csv.gz')]
  
  
  def to_date(x):
      """Convert to timestamp."""
      import pandas as pd
      try:
          ts = pd.Timestamp(x.split()[0])
          return ts
      except BaseException:
          pass
      return x
  
  
  #########################################################################################
  # --- now set up the chains and links, based on configuration flags
  
  # --- example 2: readdata loops over the input files, with file chunking.
  
  if settings['do_loop']:
      ch = Chain('Data')
  
      # --- a loop is set up in the chain MyChain.
      #     we iterate over (chunks of) the next file in the list until the iterator is done.
      #     then move on to the next chain (Overview)
  
      # --- readdata keeps on opening the next 400 lines of the open or next file in the file list.
      #     all kwargs are passed on to pandas file reader.
      read_data = analysis.ReadToDf(name='dflooper', key='rc', reader='csv')
      read_data.chunksize = chunk_size
      read_data.path = input_files
      ch.add(read_data)
  
      # add conversion functions to "Data" chain
      # here, convert column 'registered', an integer, to an actual timestamp.
      conv_funcs = [{'func': to_date, 'colin': 'registered', 'colout': 'date'}]
      transform = analysis.ApplyFuncToDf(name='Transform', read_key=read_data.key,
                                         apply_funcs=conv_funcs)
      ch.add(transform)
  
      # --- As an example, will fill histogram iteratively over the file loop
      hf = analysis.HistogrammarFiller()
      hf.read_key = 'rc'
      hf.store_key = 'hist'
      hf.logger.log_level = LogLevel.DEBUG
      # colums that are picked up to do value_counting on in the input dataset
      # note: can also be 2-dim: ['isActive','age']
      # in this example, the rest are one-dimensional histograms
      hf.columns = ['date', 'isActive', 'age', 'eyeColor', 'gender', 'company',
                    'latitude', 'longitude', ['isActive', 'age'], ['latitude', 'longitude']]
      # binning is apply to all input columns that are numeric or timestamps.
      # default binning is: bin_width = 1, bin_offset = 0
      # for timestamps, default binning is: { 'bin_width': np.timedelta64(30,'D'),
      #                                       'bin_offset': np.datetime64('2010-01-04') } }
      hf.bin_specs = {'longitude': {'bin_width': 5, 'bin_offset': 0},
                      'latitude': {'bin_width': 5, 'bin_offset': 0}}
      ch.add(hf)
  
      # --- this serves as the continue statement of the loop. go back to start of the chain.
      repeater = core_ops.RepeatChain()
      # repeat until readdata says halt.
      repeater.listen_to = 'chainRepeatRequestBy_' + read_data.name
      ch.add(repeater)
  
      link = core_ops.DsObjectDeleter()
      link.keep_only = ['hist', 'n_sum_rc', 'rc']
      ch.add(link)
  
  # --- print contents of the datastore
  if settings['do_plotting']:
      ch = Chain('Overview')
      pds = core_ops.PrintDs(name='End')
      pds.keys = ['n_sum_rc']
      ch.add(pds)
  
      # --- make a nice summary report of the created histograms
      hist_summary = visualization.DfSummary(name='HistogramSummary',
                                             read_key=hf.store_key)
      ch.add(hist_summary)
  
  #########################################################################################
  
  logger.debug('Done parsing configuration file esk303_hgr_filler_plotter.py.')
  