text: #
  # Licensed to the Apache Software Foundation (ASF) under one or more
  # contributor license agreements.  See the NOTICE file distributed with
  # this work for additional information regarding copyright ownership.
  # The ASF licenses this file to You under the Apache License, Version 2.0
  # (the "License"); you may not use this file except in compliance with
  # the License.  You may obtain a copy of the License at
  #
  #    http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  
  import logging
  import os
  import subprocess
  import csv
  import sys
  import ConfigParser
  
  class Util(object):
  	
  	@classmethod
  	def get_logger(cls,logger_name,create_file=False):
  		
  
  		# create logger for prd_ci
  		log = logging.getLogger(logger_name)
  		log.setLevel(level=logging.INFO)
  		
  		# create formatter and add it to the handlers
  		formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
  		
  		if create_file:
  				# create file handler for logger.
  				fh = logging.FileHandler('oa.log')
  				fh.setLevel(level=logging.DEBUG)
  				fh.setFormatter(formatter)
  		# reate console handler for logger.
  		ch = logging.StreamHandler()
  		ch.setLevel(level=logging.DEBUG)
  		ch.setFormatter(formatter)
  
  		# add handlers to logger.
  		if create_file:
  			log.addHandler(fh)
  
  		log.addHandler(ch)
  		return  log
  
  	@classmethod
  	def get_spot_conf(cls):
  		
  		conf_file = "/etc/spot.conf"
  		config = ConfigParser.ConfigParser()
  		config.readfp(SecHead(open(conf_file)))	
  
  		return config
  	
  	@classmethod
  	def create_oa_folders(cls,type,date):		
  
  		# create date and ingest summary folder structure if they don't' exist.
  		root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  		data_type_folder = "{0}/data/{1}/{2}"
  		if not os.path.isdir(data_type_folder.format(root_path,type,date)): os.makedirs(data_type_folder.format(root_path,type,date))
  		if not os.path.isdir(data_type_folder.format(root_path,type,"ingest_summary")): os.makedirs(data_type_folder.format(root_path,type,"ingest_summary"))
  
  		# create ipynb folders.
  		ipynb_folder = "{0}/ipynb/{1}/{2}".format(root_path,type,date)
  		if not os.path.isdir(ipynb_folder): os.makedirs(ipynb_folder)
  
  		# retun path to folders.
  		data_path = data_type_folder.format(root_path,type,date)
  		ingest_path = data_type_folder.format(root_path,type,"ingest_summary")		
  		return data_path,ingest_path,ipynb_folder
  	
  	@classmethod
  	def get_ml_results_form_hdfs(cls,hdfs_file_path,local_path):
  
  		# get results from hdfs.
  		get_results_cmd = "hadoop fs -get {0} {1}/.".format(hdfs_file_path,local_path)
  		subprocess.call(get_results_cmd,shell=True)
  		return get_results_cmd
  
  	@classmethod
  	def read_results(cls,file,limit, delimiter=','):
  		
  		# read csv results.
  		result_rows = []
  		with open(file, 'rb') as results_file:
  			csv_reader = csv.reader(results_file, delimiter = delimiter)
  			for i in range(0, int(limit)):
  				try:
  					row = csv_reader.next()
  				except StopIteration:
  					return result_rows
  				result_rows.append(row)
  		return result_rows
  
  	@classmethod
  	def ip_to_int(self,ip):
  		
  		try:
  			o = map(int, ip.split('.'))
  			res = (16777216 * o[0]) + (65536 * o[1]) + (256 * o[2]) + o[3]
  			return res    
  
  		except ValueError:
  			return None
  	
  	
  	@classmethod
  	def create_csv_file(cls,full_path_file,content,delimiter=','):   
  		with open(full_path_file, 'w+') as u_file:
  			writer = csv.writer(u_file, quoting=csv.QUOTE_NONE, delimiter=delimiter)
  			writer.writerows(content)
  
  
  class SecHead(object):
      def __init__(self, fp):
          self.fp = fp
          self.sechead = '[conf]\n'
  
      def readline(self):
          if self.sechead:
              try: 
                  return self.sechead
              finally: 
                  self.sechead = None
          else: 
              return self.fp.readline()
  
  class ProgressBar(object):
  
  	def __init__(self,total,prefix='',sufix='',decimals=2,barlength=60):
  
  		self._total = total
  		self._prefix = prefix
  		self._sufix = sufix
  		self._decimals = decimals
  		self._bar_length = barlength
  		self._auto_iteration_status = 0
  
  	def start(self):
  
  		self._move_progress_bar(0)
  	
  	def update(self,iterator):
  		
  		self._move_progress_bar(iterator)
  
  	def auto_update(self):
  
  		self._auto_iteration_status += 1		
  		self._move_progress_bar(self._auto_iteration_status)
  	
  	def _move_progress_bar(self,iteration):
  
  		filledLength    = int(round(self._bar_length * iteration / float(self._total)))
  		percents        = round(100.00 * (iteration / float(self._total)), self._decimals)
  		bar             = '#' * filledLength + '-' * (self._bar_length - filledLength)	
  		sys.stdout.write("{0} [{1}] {2}% {3}\r".format(self._prefix, bar, percents, self._sufix))		
  		sys.stdout.flush()
  		
  		if iteration == self._total:print("\n")
  
  		
  	
  
  