code: def test_list_paths(s3root, prefixes, expected):
      def urls_by_prefix(prefix):
          root = os.path.join(s3root, prefix)
          for url, v in expected.items():
              if url.startswith(root) and v[None].size is not None:
                  yield url
  
      # 1) test that list_paths() without arguments works
      matches = {
          prefix: frozenset(urls_by_prefix(prefix))
          for prefix in prefixes
      }
      non_empty = {prefix for prefix, urls in matches.items() if urls}
  
      with S3(s3root=s3root) as s3:
          s3objs = s3.list_paths()
          # found_prefixes is a subset of paths under s3root
          found_prefixes = [e for e in s3objs if e.key in prefixes]
          # we expect to find all non-empty prefixes under the s3root
          assert {e.key for e in found_prefixes} == non_empty
          # they should be all marked as non-existent objects, just prefixes
          assert all(not e.exists for e in found_prefixes)
          # they should be all marked as not downloaded
          assert all(not e.downloaded for e in found_prefixes)
  
      # 2) test querying by many prefixes
      with S3(s3root=s3root) as s3:
          s3objs = s3.list_paths(prefixes)
          assert (
              frozenset(
                  e.prefix.rstrip("/").split("/")[-1] for e in s3objs
              )
              == non_empty
          )
  
          for prefix, exp in matches.items():
              exists = frozenset(
                  e.url
                  for e in s3objs
                  if e.prefix == prefix and e.exists
              )
              not_exists = frozenset(
                  e.url
                  for e in s3objs
                  if e.prefix == prefix and not e.exists
              )
              # every object should be expected
              assert all(e in exp for e in exists)
              # not existing ones are prefixes, they shouldn't match
              assert all(e not in exp for e in not_exists)
  
      # 3) eventually list_paths should hit the leaf
      for url, v in expected.items():
          if v[None].size is None:
              with S3() as s3:
                  # querying a non-existent object should return
                  # prefixes or nothing
                  s3objs = s3.list_paths([url])
                  assert [e for e in s3objs if e.exists] == []
          else:
              suffix = url[len(s3root) :]
              expected_keys = suffix.split("/")
              if len(expected_keys) > 20:
                  # speed optimization: exclude crazy long paths
                  continue
              got_url = s3root
              for idx, expected_key in enumerate(expected_keys):
                  with S3(s3root=got_url) as s3:
                      s3objs = s3.list_paths()
                      # are we at the leaf?
                      if idx == len(expected_keys) - 1:
                          # a leaf object should always exist
                          [match] = [
                              e
                              for e in s3objs
                              if e.key == expected_key and e.exists
                          ]
                      else:
                          # a non-leaf may match objects that are also prefixes
                          [match] = [
                              e
                              for e in s3objs
                              if e.key == expected_key and not e.exists
                          ]
                      # prefix + key == url
                      assert os.path.join(
                          match.prefix, match.key
                      ) == match.url.rstrip("/")
                      got_url = match.url
  
              # the leaf should be the object itself
              assert match.url == url
  
imports: import os
  from re import I
  import shutil
  from hashlib import sha1
  from tempfile import mkdtemp
  from itertools import groupby, starmap
  import random
  from uuid import uuid4
  from metaflow.plugins.datatools import s3
  import pytest
  from metaflow import current, namespace, Run
  from metaflow.plugins.datatools.s3 import S3, MetaflowS3AccessDenied, MetaflowS3NotFound, MetaflowS3URLException, MetaflowS3InvalidObject, S3PutObject, S3GetObject
  from metaflow.util import to_bytes, unicode_type
  from .None import s3_data
  from ..None import FakeFlow, DO_TEST_RUN
label_desc: Use image node_id 21 for calling a global function with example usage: test_list_paths(s3root, prefixes, expected) without return types