code: def get_all_website_links(url):
      logger.info("crawling: {url} ...".format(url=url))
      urls = set()
  
      domain_name = urlparse(url).netloc
      try:
          # Create a session to handle cookies
          session = requests.Session()
  
          # Add headers to mimic a real browser request
          headers = {
              "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"
          }
  
          # Send GET request
          response = session.get(url, headers=headers)
          # Accept cookies if necessary
          if "cookie" in response.request.headers:
              session.cookies.update(response.cookies)
  
          content_type = response.headers.get(
              "Content-Type", ""
          ).lower()
  
          if "application/pdf" in content_type:
              content_html = "PDF"
              content_text = pdf_to_markdown(response)
          else:
              content_html = response.text
  
              # Parse HTML content with BeautifulSoup
              soup = BeautifulSoup(content_html, "html.parser")
              content_text = get_readable_text_from_soup(soup)
              for a_tag in soup.findAll("a"):
                  href = a_tag.attrs.get("href")
                  if href == "" or href is None:
                      continue
                  href = urljoin(url, href)
                  parsed_href = urlparse(href)
  
                  href = (
                      parsed_href.scheme
                      + "://"
                      + parsed_href.netloc
                      + parsed_href.path
                  )
                  if not is_valid(href):
                      continue
                  if href in urls:
                      continue
                  if domain_name != parsed_href.netloc:
                      continue
  
                  href = href.rstrip("/")
                  urls.add(href)
  
      except Exception as e:
          error_message = traceback.format_exc().splitlines()[-1]
          logger.error("An exception occurred: %s", str(e))
          return {
              "url": url,
              "urls": urls,
              "html_content": "",
              "text_content": "",
              "error": str(error_message),
          }
  
      return {
          "url": url,
          "urls": urls,
          "html_content": content_html,
          "text_content": content_text,
          "error": None,
      }
  
imports: import concurrent.futures
  import io
  import re
  import traceback
  from threading import Lock
  from urllib.parse import urljoin, urlparse
  import fitz
  import pandas
  import requests
  from bs4 import BeautifulSoup
  from mindsdb.utilities import log
label_desc: Use image node_id 4 for calling a global function with example usage: get_all_website_links(url) and returns: dict, dict