text: """Helper functions which don't fit anywhere else"""
  import ast
  import inspect
  import os
  import re
  import hashlib
  import warnings
  from collections import deque
  from contextlib import contextmanager
  from importlib import import_module
  from pkgutil import iter_modules
  from textwrap import dedent
  
  from w3lib.html import replace_entities
  
  from scrapy.utils.datatypes import LocalWeakReferencedCache
  from scrapy.utils.python import flatten, to_unicode
  from scrapy.item import _BaseItem
  from scrapy.utils.deprecate import ScrapyDeprecationWarning
  
  
  _ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes
  
  
  def arg_to_iter(arg):
      """Convert an argument to an iterable. The argument can be a None, single
      value, or an iterable.
  
      Exception: if arg is a dict, [arg] will be returned
      """
      if arg is None:
          return []
      elif not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'):
          return arg
      else:
          return [arg]
  
  
  def load_object(path):
      """Load an object given its absolute object path, and return it.
  
      The object can be the import path of a class, function, variable or an
      instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'.
  
      If ``path`` is not a string, but is a callable object, such as a class or
      a function, then return it as is.
      importlib.import_module (module,package) 返回的是相应的模块，package仅在相对导入时候引入
      如果path参数不是str ，检查是否是函数 是返回 不是则报类型错
      str.rindex 是从右边数的index
  
      本函数是返回要么是这个对象，要么从str载入的对象
      """
  
      if not isinstance(path, str):
          if callable(path):
              return path
          else:
              raise TypeError("Unexpected argument type, expected string "
                              "or object, got: %s" % type(path))
  
      try:
          dot = path.rindex('.')
      except ValueError:
          raise ValueError(f"Error loading object '{path}': not a full path")
  
      module, name = path[:dot], path[dot + 1:]
      mod = import_module(module)
  
      try:
          obj = getattr(mod, name)
      except AttributeError:
          raise NameError(f"Module '{module}' doesn't define any object named '{name}'")
  
      return obj
  
  
  def walk_modules(path):
      """Loads a module and all its submodules from the given module path and
      returns them. If *any* module throws an exception while importing, that
      exception is thrown back.
  
      For example: walk_modules('scrapy.utils') 返回给定路径/或者模块下的内所有的模组
      """
  
      mods = []
      mod = import_module(path)
      mods.append(mod)
      if hasattr(mod, '__path__'):# __path__是当前包的路径文件
          #pkgutil.iter_modules 是当给定一个string类型的 绝对地址 然后将改地址下所有的子模块返回 参数是一个FileFinder对象，第二个是模块名字 第三个是是否是pakge的标志
          for _, subpath, ispkg in iter_modules(mod.__path__): #仅包本身有__path__属性 其他函数没有
              fullpath = path + '.' + subpath
              if ispkg: #递归调用
                  mods += walk_modules(fullpath)
              else:
                  submod = import_module(fullpath)#动态加载模块 相当于import
                  mods.append(submod)
      return mods
  
  
  def extract_regex(regex, text, encoding='utf-8'):
      """Extract a list of unicode strings from the given text/encoding using the following policies:
  
      * if the regex contains a named group called "extract" that will be returned
      * if the regex contains multiple numbered groups, all those will be returned (flattened)
      * if the regex doesn't contain any group the entire regex matching is returned
      """
      warnings.warn(
          "scrapy.utils.misc.extract_regex has moved to parsel.utils.extract_regex.",
          ScrapyDeprecationWarning,
          stacklevel=2
      )
  
      if isinstance(regex, str):
          regex = re.compile(regex, re.UNICODE)
  
      try:
          strings = [regex.search(text).group('extract')]   # named group
      except Exception:
          strings = regex.findall(text)    # full regex or numbered groups
      strings = flatten(strings)
  
      if isinstance(text, str):
          return [replace_entities(s, keep=['lt', 'amp']) for s in strings]
      else:
          return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])
                  for s in strings]
  
  
  def md5sum(file):
      """Calculate the md5 checksum of a file-like object without reading its
      whole content in memory.
  
      >>> from io import BytesIO
      >>> md5sum(BytesIO(b'file content to hash'))
      '784406af91dd5a54fbb9c84c2236595a'
      """
      m = hashlib.md5()
      while True:
          d = file.read(8096)
          if not d:
              break
          m.update(d)
      return m.hexdigest()
  
  
  def rel_has_nofollow(rel):
      """Return True if link rel attribute has nofollow type"""
      return rel is not None and 'nofollow' in rel.split()
  
  
  def create_instance(objcls, settings, crawler, *args, **kwargs):
      """Construct a class instance using its ``from_crawler`` or
      ``from_settings`` constructors, if available.
  
      At least one of ``settings`` and ``crawler`` needs to be different from
      ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.
      If ``crawler`` is ``None``, only the ``from_settings`` constructor will be
      tried.
  
      ``*args`` and ``**kwargs`` are forwarded to the constructors.
  
      Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
  
      .. versionchanged:: 2.2
         Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
         extension has not been implemented correctly).
         他这里就是一个实例化 优先使用from_crawler 其次是from_settings
         最后直接调用 objcls(*args, **kwargs)来实例化一个对象
      """
      if settings is None: #从crawler的setting中拿到setting对象
          if crawler is None:
              raise ValueError("Specify at least one of settings and crawler.")
          settings = crawler.settings
      if crawler and hasattr(objcls, 'from_crawler'): #如果实例化 目标类 中有from_crawler方法 就用这个方法实例化
          instance = objcls.from_crawler(crawler, *args, **kwargs)
          method_name = 'from_crawler'
      elif hasattr(objcls, 'from_settings'): #如果 目标类 中有from_settings方法 就用这个方法实例化目标
          instance = objcls.from_settings(settings, *args, **kwargs)
          method_name = 'from_settings'
      else:
          instance = objcls(*args, **kwargs) # 前面都没有 普通方法实例化
          method_name = '__new__'
      if instance is None:
          raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
      return instance
  
  
  @contextmanager
  def set_environ(**kwargs):
      """Temporarily set environment variables inside the context manager and
      fully restore previous environment afterwards
      """
  
      original_env = {k: os.environ.get(k) for k in kwargs}
      os.environ.update(kwargs)
      try:
          yield
      finally:
          for k, v in original_env.items():
              if v is None:
                  del os.environ[k]
              else:
                  os.environ[k] = v
  
  
  def walk_callable(node):
      """Similar to ``ast.walk``, but walks only function body and skips nested
      functions defined within the node.
      """
      todo = deque([node])
      walked_func_def = False
      while todo:
          node = todo.popleft()
          if isinstance(node, ast.FunctionDef):
              if walked_func_def:
                  continue
              walked_func_def = True
          todo.extend(ast.iter_child_nodes(node))
          yield node
  
  
  _generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
  
  
  def is_generator_with_return_value(callable):
      """
      Returns True if a callable is a generator function which includes a
      'return' statement with a value different than None, False otherwise
      """
      if callable in _generator_callbacks_cache:
          return _generator_callbacks_cache[callable]
  
      def returns_none(return_node):
          value = return_node.value
          return value is None or isinstance(value, ast.NameConstant) and value.value is None
  
      if inspect.isgeneratorfunction(callable):
          tree = ast.parse(dedent(inspect.getsource(callable)))
          for node in walk_callable(tree):
              if isinstance(node, ast.Return) and not returns_none(node):
                  _generator_callbacks_cache[callable] = True
                  return _generator_callbacks_cache[callable]
  
      _generator_callbacks_cache[callable] = False
      return _generator_callbacks_cache[callable]
  
  
  def warn_on_generator_with_return_value(spider, callable):
      """
      Logs a warning if a callable is a generator function and includes
      a 'return' statement with a value different than None
      """
      if is_generator_with_return_value(callable):
          warnings.warn(
              f'The "{spider.__class__.__name__}.{callable.__name__}" method is '
              'a generator and includes a "return" statement with a value '
              'different than None. This could lead to unexpected behaviour. Please see '
              'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement '
              'for details about the semantics of the "return" statement within generators',
              stacklevel=2,
          )
  