text: #!/usr/bin/env python
  # license removed for brevity
  import sys
  import os
  current_folder = os.path.dirname(os.path.realpath(__file__))
  sys.path.append(current_folder)
  
  import time
  import threading
  from multiprocessing import Queue
  
  from config.console_formatter import Console_Formatter
  from modules.HTTP_Requestor import HTTP_Requestor
  from modules.HTTP_Parser import HTTP_Parser
  
  
  class Spider:
      url = None
      saving_directory = os.path.join(current_folder, "Spider_Fetch_Data")
  
      http_requestor_ = HTTP_Requestor(use_info=True)
      http_parser_ = HTTP_Parser(use_info=True)
      console_formatter_ = Console_Formatter()
      program_name_ = __name__
      
      def __init__(self, url=None, use_info=True, encoding='utf-8'):
          #self.set_encoding(encoding) #only for python2.x
          self.url = url
          self.use_info = use_info
          
      def init(self):
          if not os.path.exists(self.saving_directory):
              os.mkdir(self.saving_directory)
      '''
      def test(self, url):
          r = self.url_downloader(url, "temp.html")
          self.http_parser_.set_html_data(r.text)
          links = self.http_parser_.link_finder()
          for link in links:
              print(link.attrs.items()[0][1])
      '''        
      def set_encoding(self, encoding='utf-8'):
          reload(sys)
          sys.setdefaultencoding(encoding)
      
      def image_downloader(self, url, file_name=None):
          r = self.http_requestor_.get(url=url, payload=None, use_parse=False, stream=False)
          self.http_parser_.set_html_data(r.text)
          imgs = self.http_parser_.image_finder()
          if imgs == None:    return False
          
          for img in imgs:
              for key in img.attrs.keys():
                  if key.find('src') > 0:
                      img_url = img.get(key)
                      break
              
              self.http_requestor_.use_info = False
              r = self.http_requestor_.get(url=img_url, use_parse=False, stream=False)
              if r == None:
                  if self.use_info : 
                      msg = "Downloading image : {} fail !".format(img_url)
                      print(self.console_formatter_.WARN(self.program_name_, msg))
              else:
                  image_file = os.path.join(self.saving_directory, img_url.split('/')[-1] if file_name == None else file_name)
                  with open(image_file, 'wb') as fid:
                      fid.write(r.content)
                      fid.close()
                  if self.use_info : 
                      msg = "Downloading image : {} >> {}".format(img_url, image_file)
                      print(self.console_formatter_.INFO(self.program_name_, msg))
              
      def url_downloader(self, url, file_name=None, stream=True):
          r = self.http_requestor_.get(url=url, payload=None, use_parse=False, stream=stream)
          if r == None:
              if self.use_info : 
                  msg = "Downloading url : {} fail !".format(url)
                  print(self.console_formatter_.WARN(self.program_name_, msg))
              return None
          else:
              url_file = os.path.join(self.saving_directory, file_name)
              with open(url_file, 'w') as fid:
                  fid.write(r.text)
                  fid.close()
              if self.use_info : 
                  msg = "Downloading url : {} >> {}".format(url, url_file)
                  print(self.console_formatter_.INFO(self.program_name_, msg))
              return r
  