class_docstr: LinkContentFetcher is a component for fetching and extracting content from URLs. It supports handling various
  content types, retries on failures, and automatic user-agent rotation for failed web requests.
class_docstr_tok: ["LinkContentFetcher","is","a","component","for","fetching","and","extracting","content","from","URLs",".","It","supports","handling","various","content","types",",","retries","on","failures",",","and","automatic","user-agent","rotation","for","failed","web","requests","."]
docstr: Initializes a LinkContentFetcher instance.
  
  :param raise_on_failure: If True, raises an exception if it fails to fetch a single URL.
      For multiple URLs, it logs errors and returns the content it successfully fetched. Default is True.
  :param user_agents: A list of user agents for fetching content. If None, a default user agent is used.
  :param retry_attempts: Specifies how many times you want it to retry to fetch the URL's content. Default is 2.
  :param timeout: Timeout in seconds for the request. Default is 3.
docstr_tok: ["Initializes","a","LinkContentFetcher","instance",".",":","param","raise_on_failure",":","If","True",",","raises","an","exception","if","it","fails","to","fetch","a","single","URL",".","For","multiple","URLs",",","it","logs","errors","and","returns","the","content","it","successfully","fetched",".","Default","is","True",".",":","param","user_agents",":","A","list","of","user","agents","for","fetching","content",".","If","None",",","a","default","user","agent","is","used",".",":","param","retry_attempts",":","Specifies","how","many","times","you","want","it","to","retry","to","fetch","the","URL","'s","content",".","Default","is","2.",":","param","timeout",":","Timeout","in","seconds","for","the","request",".","Default","is","3","."]
code: def __init__(
      self,
      raise_on_failure: bool = True,
      user_agents: Optional[List[str]] = None,
      retry_attempts: int = 2,
      timeout: int = 3,
  ):
      """
      Initializes a LinkContentFetcher instance.
  
      :param raise_on_failure: If True, raises an exception if it fails to fetch a single URL.
          For multiple URLs, it logs errors and returns the content it successfully fetched. Default is True.
      :param user_agents: A list of user agents for fetching content. If None, a default user agent is used.
      :param retry_attempts: Specifies how many times you want it to retry to fetch the URL's content. Default is 2.
      :param timeout: Timeout in seconds for the request. Default is 3.
      """
      self.raise_on_failure = raise_on_failure
      self.user_agents = user_agents or [DEFAULT_USER_AGENT]
      self.current_user_agent_idx: int = 0
      self.retry_attempts = retry_attempts
      self.timeout = timeout
  
      # register default content handlers that extract data from the response
      self.handlers: Dict[
          str, Callable[[Response], ByteStream]
      ] = defaultdict(lambda: text_content_handler)
      self.handlers["text/html"] = text_content_handler
      self.handlers["text/plain"] = text_content_handler
      self.handlers["application/pdf"] = binary_content_handler
      self.handlers["application/octet-stream"] = binary_content_handler
  
      @retry(
          reraise=True,
          stop=stop_after_attempt(self.retry_attempts),
          wait=wait_exponential(multiplier=1, min=2, max=10),
          retry=(
              retry_if_exception_type(
                  (HTTPError, requests.RequestException)
              )
          ),
          # This method is invoked only after failed requests (exception raised)
          after=self._switch_user_agent,
      )
      def get_response(url):
          # we need to copy because we modify the headers
          headers = REQUEST_HEADERS.copy()
          headers["User-Agent"] = self.user_agents[
              self.current_user_agent_idx
          ]
          response = requests.get(
              url, headers=headers, timeout=timeout or 3
          )
          response.raise_for_status()
          return response
  
      self._get_response: Callable = get_response
  
file_path: haystack/haystack/components/fetchers/link_content.py
imports: import logging
  from collections import defaultdict
  from concurrent.futures import ThreadPoolExecutor
  from typing import Callable, Dict, List, Optional, Tuple
  import requests
  from requests import Response
  from requests.exceptions import HTTPError
  from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential
  from haystack import component
  from haystack.dataclasses import ByteStream
  from haystack.version import __version__
label_desc: Use image node_id 1 to create a new LinkContentFetcher object  with example: obj = LinkContentFetcher(raise_on_failure, user_agents, retry_attempts, timeout)