text: import requests
  from bs4 import BeautifulSoup
  import time
  from fake_useragent import UserAgent
  import ast
  import os
  import logging
  import random
  from lxml import html
  
  ua = UserAgent()
  logger = logging.getLogger(__name__)
  
  
  class ByRequest():
      proxies_retries = {
          None: 3,
          "crawlera": 3,
          "scrapoxy": 3,
          "luminati": 3
      }
      timeout = 15
      extras = {}
      delay = [1, 3]
      max_retries = 2
      proxies_order = [None, "crawlera", "scrapoxy", "luminati"]
      headers = {}
      cookies = {}
      verify = True
  
      def __init__(self, proxies=False, max_retries=False, cookies=False, fake_ua=True, headers=False, timeout=False,
                   delay=False, verify=True):
          """
          Create object with the initial parameters
          :param proxies:
              * list of proxy services in execution order ["1st proxy service", "2nd", "3rd", ...]
              * str of the only proxy service that will be used
              * dict of proxy services in priority order {
                                                   "1st proxy service" : <<number of tries>>,
                                                   "2nd proxy service" : <<number of tries>>,
                                                   ..} ** The number of tries will overwrite the max_retries param
              *** Use None to use no proxy server
          :param max_retries: integer of max number of retries by failed request for every proxy service
          :param cookies: RequestCookieJar or Dict with the cookies that will persist for the whole request session
          :param fake_ua: Boolean to indicate if a Fake User-Agent will be used for the whole request session
              ** This User-Agent will overwrite the one from the headers
          :param headers: Dict with the headers that will persist for the whole session
          :param timeout: Integer or String digit with the timeout for the request that will persist for the whole session
          :param delay:
              * list [min, max] of the range of random seconds of wait between a request failed and a new retry
              * tuple (min, max) of the range of random seconds of wait between a request failed and a new retry
              * int or string digit of the max seconds of wait after a request failed
          :param verify: Boolean to indicate if the SSL verification is enabled or not
          """
          if max_retries:
              logger.debug("Assigning max_retries...")
              try:
                  self.max_retries = int(max_retries)
              except:
                  logger.error("Max retires cannot be converted to integer")
              for proxy in self.proxies_retries:
                  self.proxies_retries[proxy] = self.max_retries
  
          if proxies:
              logger.debug("Assigning proxies...")
              if isinstance(proxies, list):
                  logger.debug("Assigning order of proxy servers...")
                  proxy_order = []
                  for proxy in proxies:
                      if proxy in self.proxies_retries.keys():
                          proxy_order.append(proxy)
                      elif isinstance(proxy, str) and proxy.lower() in self.proxies_retries.keys():
                          proxy_order.append(proxy.lower())
                  self.proxies_order = proxy_order
              elif isinstance(proxies, str):
                  logger.debug("Assigning single proxy server...")
                  if proxies.lower() in self.proxies_order:
                      self.proxies_order = [proxies.lower()]
                  else:
                      logger.error("{proxy} is not a valid proxy server".format(proxy=proxies))
              elif isinstance(proxies, dict):
                  logger.debug("Assigning order and max retries of proxy servers...")
                  proxies_retries_aux = {}
                  for proxy, retries in proxies.items():
                      if proxy in self.proxies_retries.keys() or (
                              isinstance(proxy, str) and proxy.lower() in self.proxies_retries.keys()):
                          try:
                              proxies_retries_aux[proxy] = int(retries)
                          except Exception as e:
                              logger.warning("Error while assigning proxies")
                              logger.error(e)
                      else:
                          logger.error("{proxy} is not in the list of valid proxies".format(proxy=str(proxy)))
                  if proxies_retries_aux:
                      self.proxies_retries = proxies_retries_aux
                      self.proxies_order = list(self.proxies_retries.keys())
                      if max_retries:
                          logger.warning("Overwriting max_retries")
  
          if headers:
              logger.debug("Assigning headers...")
              if isinstance(headers, dict):
                  self.headers = headers
              elif isinstance(headers, str):
                  try:
                      self.headers = ast.literal_eval(headers)
                  except:
                      self.headers = {}
                      logger.error("Headers string cannot be converted to dict")
              else:
                  logger.error("Headers are not valid")
  
          if fake_ua:
              logger.debug("Assigning fake User-Agent...")
              try:
                  user_agent = ua.random
                  self.headers['User-Agent'] = user_agent
              except Exception as e:
                  logger.warning("The fake user agent cannot be added to headers")
                  logger.error(e)
  
          if cookies:
              logger.debug("Assigning cookies...")
              if isinstance(cookies, requests.cookies.RequestsCookieJar) or isinstance(cookies, dict):
                  self.cookies = cookies
              elif isinstance(cookies, str):
                  try:
                      self.cookies = ast.literal_eval(cookies)
                  except:
                      logger.error("Cookies string cannot be converted to dict")
              else:
                  logger.error("Cookies are not valid")
  
          if timeout:
              logger.debug("Assigning timeout...")
              if isinstance(timeout, int):
                  self.timeout = abs(timeout)
              else:
                  try:
                      self.timeout = abs(int(timeout))
                  except:
                      logger.error("Timeout cannot be converted to integer")
  
          if delay:
              logger.debug("Assigning delay...")
              if isinstance(delay, list) or isinstance(delay, tuple):
                  if len(delay) == 2:
                      logger.debug("Assigning min & max seconds...")
                      try:
                          self.delay[0] = int(delay[0])
                          self.delay[1] = int(delay[1])
                      except:
                          logger.error("Delay values cannot converted to integers")
                  elif len(delay) == 1:
                      try:
                          logger.debug("Assigning max seconds...")
                          self.delay[1] = int(delay[0])
                      except:
                          logger.error("Delay value cannot converted to integer")
                  else:
                      logger.error("Delay should contain 1 or 2 integer values")
              else:
                  try:
                      logger.debug("Assigning max seconds...")
                      self.delay[1] = int(delay)
                  except:
                      logger.error("Delay value cannot be converted into integer")
  
          if verify is not True:
              self.verify = verify
  
  
      def request(self, method, url, fake_ua=False, return_json=False, br_session=True,  **kwargs):
          """
          Method to create a http request
          :param method: str "GET" or "POST" to indicate the request method
          :param url: str of the url that will be requested
          :param fake_ua: boolean to indicate if a Fake UserAgent will be send on the headers
          :param return_json: Boolean to indicate if the response will be a json
          :param br_session: Boolean to indicate if the parameters of the session will be used or not
          :param kwargs: Extra parameters that can be used for requests.request
          :return: request or json depending on the return_json param.
          """
          logger.debug("Executing request...")
          if self.proxies_order:
              if br_session == True:
                  headers_ = kwargs.pop("headers", self.headers)
                  cookies_ = kwargs.pop("cookies", self.cookies)
              else:
                  headers_ = kwargs.pop("headers", {})
                  cookies_ = kwargs.pop("cookies", {})
  
              verify_ = kwargs.pop("verify", self.verify)
              timeout_ = kwargs.pop("timeout", self.timeout)
              proxies_ = kwargs.pop("proxies", False)
              if proxies_ == False:
                  is_proxies_defined = False
              else:
                  is_proxies_defined = True
  
              for proxy in self.proxies_order:
                  logger.debug("Trying with Proxy server {proxy}...".format(proxy=proxy))
                  for retry in range(1, self.proxies_retries.get(proxy) + 1):
                      logger.debug("Try #{retry}...".format(retry=retry))
                      try:
                          if not is_proxies_defined:
                              proxies_ = self.get_proxies(proxy)
                          if fake_ua:
                              headers_["User-Agent"] = ua.random
                          if retry == self.proxies_retries.get(proxy) and self.verify == False:
                              logger.warning("Trying with verify as False")
                              response = requests.request(method, url, headers=headers_, proxies=proxies_,
                                                          cookies=cookies_, verify=True, timeout=timeout_, **kwargs)
                          else:
                              print("Headers --->", str(headers_), "proxies --->", str(proxies_))
                              response = requests.request(method, url, headers=headers_, proxies=proxies_,
                                                          cookies=cookies_, verify=verify_, timeout=timeout_, **kwargs)
                          if response.status_code == 200:
                              if not return_json:
                                  return response
                              else:
                                  try:
                                      return response.json()
                                  except:
                                      logger.error("Json cannot be obtained from request")
                                      return response
  
                          else:
                              logger.warning(
                                  "[{proxy}] The Request #{retry} failed: {url}".format(proxy=proxy, retry=retry,
                                                                                        url=url))
                              time.sleep(random.randrange(self.delay[0], self.delay[1]))
                              continue
  
                      except Exception as e:
                          logger.error(e)
                          logger.warning(
                              "[{proxy}] The Request #{retry} failed: {url}".format(proxy=proxy, retry=retry, url=url))
                          time.sleep(random.randrange(self.delay[0], self.delay[1]))
                          continue
                  logger.warning(
                      "[{proxy}] Was not able to return a good response for {url}".format(proxy=proxy, url=url))
              logger.error(
                  "REQUEST ERROR {url}".format(url=url))
              return False
          else:
              logger.error("Proxies are not well defined")
  
      def post(self, url, fake_ua=False, return_json=False, br_session=True, **kwargs):
          """
          Method to call request wiht POST as a method
          :param url: str of the url that will be requested
          :param fake_ua: boolean to indicate if a Fake UserAgent will be send on the headers
          :param return_json: Boolean to indicate if the response will be a json
          :param br_session: Boolean to indicate if the parameters of the session will be used or not
          :param kwargs: Extra parameters that can be used for requests.request
          :return: request or json depending on the return_json param.
          """
          return self.request('POST', url, return_json=return_json, fake_ua=fake_ua, br_session=br_session, **kwargs)
  
      def get(self, url, fake_ua=False, return_json=False, br_session=True, **kwargs):
          """
          Method to call request wiht GET as a method
          :param url: str of the url that will be requested
          :param fake_ua: boolean to indicate if a Fake UserAgent will be send on the headers
          :param return_json: Boolean to indicate if the response will be a json
          :param br_session: Boolean to indicate if the parameters of the session will be used or not
          :param kwargs: Extra parameters that can be used for requests.request
          :return: request or json depending on the return_json param.
          """
          return self.request('GET', url, return_json=return_json, fake_ua=fake_ua, br_session=br_session, **kwargs)
  
      def soup(self, url, **kwargs):
          """
          Method to parse the soup from an specific url
          :param url: str of the url of the html
          :return: BeautifulSoup
          """
          logger.debug("Getting soup...")
          response = self.get(url, **kwargs)
          if response:
              try:
                  return BeautifulSoup(response.content, 'html.parser')
              except Exception as e:
                  logger.error("Error while parsing the soup")
                  logger.error(e)
  
          else:
              logger.error("Soup cannot be returned")
  
      def xpath(self, url, **kwargs):
          """
          Method to parse the xpath tree from an specific url
          :param url: str of the url of the html
          :return: XpathTree
          """
          logger.debug("Getting xpath...")
          response = self.get(url, **kwargs)
          if response:
              try:
                  tree = html.fromstring(response.content)
                  return tree
              except Exception as e:
                  logger.error("Error while parsing the xpath")
                  logger.error(e)
  
          else:
              logger.error("Soup cannot be returned")
  
      @staticmethod
      def get_proxies(server=None):
          """
          Method to get the proxies request parameter for different proxy services
          :param server: Name of the proxy service
          :return: dict of http methods and their corresponding proxy service
          """
          logger.debug("Getting proxies dict...")
          if server == None:
              logger.debug("Without proxies...")
              return {}
          if server == 'crawlera':
              logger.debug("Proxies from crawlera...")
              proxy_host = "proxy.crawlera.com"
              proxy_port = "8010"
              proxy_auth = "{}:".format(os.getenv('CRAWLERA', 'api_key'))  # Make sure to include ':' at the end
              proxies = {"https": "https://{}@{}:{}/".format(proxy_auth, proxy_host, proxy_port),
                         "http": "http://{}@{}:{}/".format(proxy_auth, proxy_host, proxy_port)}
              logger.debug("Returning:" + str(proxies))
              return proxies
          if server == "scrapoxy":
              logger.debug("Proxies from scrapoy...")
              proxy_host = "{}".format(os.getenv('SCRAPOXY', 'localhost'))
              proxy_port = "8888"
              proxies = {"https": "https://{}:{}/".format(proxy_host, proxy_port),
                         "http": "http://{}:{}/".format(proxy_host, proxy_port)}
              logger.debug("Returning:" + str(proxies))
              return proxies
          if server == "luminati":
              logger.debug("Proxies from scrapoy...")
              #luminati = http://lum-customer-{costumer}-zone-{zone}:{password}@zproxy.lum-superproxy.io:{port}#
              luminati = "{}".format(os.getenv('LUMINATI', 'localhost'))
              proxies = {"https": luminati,
                         "http": luminati}
              logger.debug("Returning:" + str(proxies))
              return proxies
  
  