code: def main():
      port = get_safe_port()
  
      class Crawler(CrawlSpider):
          name = "broken-links"
          allowed_domains = ["localhost"]
          start_urls = [f"http://localhost:{port}/"]
          handle_httpstatus_list = [404]
          rules = (
              Rule(LinkExtractor(), callback="parse_item", follow=True),
          )
          links = set()
  
          def parse_item(self, response):
              if response.status == 404:
                  self.links.add(
                      (
                          response.url,
                          response.request.headers.get(
                              "Referer", None
                          ).decode("utf-8"),
                      )
                  )
  
      with server(port):
          process = CrawlerProcess(settings={"LOG_LEVEL": "ERROR"})
          process.crawl(Crawler)
          process.start()
  
      if Crawler.links:
          print("Broken links found:")
          for link, referer in Crawler.links:
              print(f"{link} in {referer}")
          sys.exit(1)
  
file_path: mlflow/docs/broken_links.py
imports: import contextlib
  import socket
  import subprocess
  import sys
  import time
  import requests
  from scrapy.crawler import CrawlerProcess
  from scrapy.linkextractors import LinkExtractor
  from scrapy.spiders import CrawlSpider, Rule
label_desc: Use image node_id 3 for calling a global function with example usage: main() without return types