text: # -*- coding: utf-8 -*-
  # Copyright 2010 Google Inc. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  """Wildcard iterator class and supporting functions."""
  
  from __future__ import absolute_import
  
  import fnmatch
  import glob
  import os
  import re
  import sys
  import textwrap
  
  from gslib.bucket_listing_ref import BucketListingBucket
  from gslib.bucket_listing_ref import BucketListingObject
  from gslib.bucket_listing_ref import BucketListingPrefix
  from gslib.cloud_api import AccessDeniedException
  from gslib.cloud_api import CloudApi
  from gslib.cloud_api import NotFoundException
  from gslib.exception import CommandException
  from gslib.storage_url import ContainsWildcard
  from gslib.storage_url import StorageUrlFromString
  from gslib.storage_url import StripOneSlash
  from gslib.storage_url import WILDCARD_REGEX
  from gslib.translation_helper import GenerationFromUrlAndString
  from gslib.util import UTF8
  
  
  FLAT_LIST_REGEX = re.compile(r'(?P<before>.*?)\*\*(?P<after>.*)')
  
  
  class WildcardIterator(object):
    """Class for iterating over Google Cloud Storage strings containing wildcards.
  
    The base class is abstract; you should instantiate using the
    wildcard_iterator() static factory method, which chooses the right
    implementation depending on the base string.
    """
  
    # TODO: Standardize on __str__ and __repr__ here and elsewhere.  Define both
    # and make one return the other.
    def __repr__(self):
      """Returns string representation of WildcardIterator."""
      return 'WildcardIterator(%s)' % self.wildcard_url.url_string
  
  
  class CloudWildcardIterator(WildcardIterator):
    """WildcardIterator subclass for buckets, bucket subdirs and objects.
  
    Iterates over BucketListingRef matching the Url string wildcard. It's
    much more efficient to first get metadata that's available in the Bucket
    (for example to get the name and size of each object), because that
    information is available in the object list results.
    """
  
    def __init__(self, wildcard_url, gsutil_api, all_versions=False,
                 debug=0, project_id=None):
      """Instantiates an iterator that matches the wildcard URL.
  
      Args:
        wildcard_url: CloudUrl that contains the wildcard to iterate.
        gsutil_api: Cloud storage interface.  Passed in for thread safety, also
                    settable for testing/mocking.
        all_versions: If true, the iterator yields all versions of objects
                      matching the wildcard.  If false, yields just the live
                      object version.
        debug: Debug level to control debug output for iterator.
        project_id: Project ID to use for bucket listings.
      """
      self.wildcard_url = wildcard_url
      self.all_versions = all_versions
      self.debug = debug
      self.gsutil_api = gsutil_api
      self.project_id = project_id
  
    def __iter__(self, bucket_listing_fields=None,
                 expand_top_level_buckets=False):
      """Iterator that gets called when iterating over the cloud wildcard.
  
      In the case where no wildcard is present, returns a single matching object,
      single matching prefix, or one of each if both exist.
  
      Args:
        bucket_listing_fields: Iterable fields to include in bucket listings.
                               Ex. ['name', 'acl'].  Iterator is
                               responsible for converting these to list-style
                               format ['items/name', 'items/acl'] as well as
                               adding any fields necessary for listing such as
                               prefixes.  API implemenation is responsible for
                               adding pagination fields.  If this is None,
                               all fields are returned.
        expand_top_level_buckets: If true, yield no BUCKET references.  Instead,
                                  expand buckets into top-level objects and
                                  prefixes.
  
      Yields:
        BucketListingRef of type BUCKET, OBJECT or PREFIX.
      """
      single_version_request = self.wildcard_url.HasGeneration()
  
      # For wildcard expansion purposes, we need at a minimum the name of
      # each object and prefix.  If we're not using the default of requesting
      # all fields, make sure at least these are requested.  The Cloud API
      # tolerates specifying the same field twice.
      get_fields = None
      if bucket_listing_fields:
        get_fields = set()
        for field in bucket_listing_fields:
          get_fields.add(field)
        bucket_listing_fields = self._GetToListFields(
            get_fields=bucket_listing_fields)
        bucket_listing_fields.update(['items/name', 'prefixes'])
        get_fields.update(['name'])
        # If we're making versioned requests, ensure generation and
        # metageneration are also included.
        if single_version_request or self.all_versions:
          bucket_listing_fields.update(['items/generation',
                                        'items/metageneration'])
          get_fields.update(['generation', 'metageneration'])
  
      # Handle bucket wildcarding, if any, in _ExpandBucketWildcards. Then
      # iterate over the expanded bucket strings and handle any object
      # wildcarding.
      for bucket_listing_ref in self._ExpandBucketWildcards(bucket_fields=['id']):
        bucket_url_string = bucket_listing_ref.url_string
        if self.wildcard_url.IsBucket():
          # IsBucket() guarantees there are no prefix or object wildcards, and
          # thus this is a top-level listing of buckets.
          if expand_top_level_buckets:
            url = StorageUrlFromString(bucket_url_string)
            for obj_or_prefix in self.gsutil_api.ListObjects(
                url.bucket_name, delimiter='/', all_versions=self.all_versions,
                provider=self.wildcard_url.scheme,
                fields=bucket_listing_fields):
              if obj_or_prefix.datatype == CloudApi.CsObjectOrPrefixType.OBJECT:
                yield self._GetObjectRef(bucket_url_string, obj_or_prefix.data,
                                         with_version=self.all_versions)
              else:  # CloudApi.CsObjectOrPrefixType.PREFIX:
                yield self._GetPrefixRef(bucket_url_string, obj_or_prefix.data)
          else:
            yield bucket_listing_ref
        else:
          # By default, assume a non-wildcarded URL is an object, not a prefix.
          # This prevents unnecessary listings (which are slower, more expensive,
          # and also subject to eventual consistency).
          if (not ContainsWildcard(self.wildcard_url.url_string) and
              self.wildcard_url.IsObject() and not self.all_versions):
            try:
              get_object = self.gsutil_api.GetObjectMetadata(
                  self.wildcard_url.bucket_name,
                  self.wildcard_url.object_name,
                  generation=self.wildcard_url.generation,
                  provider=self.wildcard_url.scheme,
                  fields=get_fields)
              yield self._GetObjectRef(
                  self.wildcard_url.bucket_url_string, get_object,
                  with_version=(self.all_versions or single_version_request))
              return
            except (NotFoundException, AccessDeniedException):
              # It's possible this is a prefix - try to list instead.
              pass
  
          # Expand iteratively by building prefix/delimiter bucket listing
          # request, filtering the results per the current level's wildcard
          # (if present), and continuing with the next component of the
          # wildcard. See _BuildBucketFilterStrings() documentation for details.
          if single_version_request:
            url_string = '%s%s#%s' % (bucket_url_string,
                                      self.wildcard_url.object_name,
                                      self.wildcard_url.generation)
          else:
            # Rstrip any prefixes to correspond with rstripped prefix wildcard
            # from _BuildBucketFilterStrings().
            url_string = '%s%s' % (bucket_url_string,
                                   StripOneSlash(self.wildcard_url.object_name)
                                   or '/')  # Cover root object named '/' case.
          urls_needing_expansion = [url_string]
          while urls_needing_expansion:
            url = StorageUrlFromString(urls_needing_expansion.pop(0))
            (prefix, delimiter, prefix_wildcard, suffix_wildcard) = (
                self._BuildBucketFilterStrings(url.object_name))
            prog = re.compile(fnmatch.translate(prefix_wildcard))
  
            # List bucket for objects matching prefix up to delimiter.
            for obj_or_prefix in self.gsutil_api.ListObjects(
                url.bucket_name, prefix=prefix, delimiter=delimiter,
                all_versions=self.all_versions or single_version_request,
                provider=self.wildcard_url.scheme,
                fields=bucket_listing_fields):
              if obj_or_prefix.datatype == CloudApi.CsObjectOrPrefixType.OBJECT:
                gcs_object = obj_or_prefix.data
                if prog.match(gcs_object.name):
                  if not suffix_wildcard or (
                      StripOneSlash(gcs_object.name) == suffix_wildcard):
                    if not single_version_request or (
                        self._SingleVersionMatches(gcs_object.generation)):
                      yield self._GetObjectRef(
                          bucket_url_string, gcs_object, with_version=(
                              self.all_versions or single_version_request))
              else:  # CloudApi.CsObjectOrPrefixType.PREFIX
                prefix = obj_or_prefix.data
                # If the prefix ends with a slash, remove it.  Note that we only
                # remove one slash so that we can successfully enumerate dirs
                # containing multiple slashes.
                rstripped_prefix = StripOneSlash(prefix)
                if prog.match(rstripped_prefix):
                  if suffix_wildcard and rstripped_prefix != suffix_wildcard:
                    # There's more wildcard left to expand.
                    url_append_string = '%s%s' % (
                        bucket_url_string, rstripped_prefix + '/' +
                        suffix_wildcard)
                    urls_needing_expansion.append(url_append_string)
                  else:
                    # No wildcard to expand, just yield the prefix
                    yield self._GetPrefixRef(bucket_url_string, prefix)
  
    def _BuildBucketFilterStrings(self, wildcard):
      """Builds strings needed for querying a bucket and filtering results.
  
      This implements wildcard object name matching.
  
      Args:
        wildcard: The wildcard string to match to objects.
  
      Returns:
        (prefix, delimiter, prefix_wildcard, suffix_wildcard)
        where:
          prefix is the prefix to be sent in bucket GET request.
          delimiter is the delimiter to be sent in bucket GET request.
          prefix_wildcard is the wildcard to be used to filter bucket GET results.
          suffix_wildcard is wildcard to be appended to filtered bucket GET
            results for next wildcard expansion iteration.
        For example, given the wildcard gs://bucket/abc/d*e/f*.txt we
        would build prefix= abc/d, delimiter=/, prefix_wildcard=d*e, and
        suffix_wildcard=f*.txt. Using this prefix and delimiter for a bucket
        listing request will then produce a listing result set that can be
        filtered using this prefix_wildcard; and we'd use this suffix_wildcard
        to feed into the next call(s) to _BuildBucketFilterStrings(), for the
        next iteration of listing/filtering.
  
      Raises:
        AssertionError if wildcard doesn't contain any wildcard chars.
      """
      # Generate a request prefix if the object name part of the wildcard starts
      # with a non-wildcard string (e.g., that's true for 'gs://bucket/abc*xyz').
      match = WILDCARD_REGEX.search(wildcard)
      if not match:
        # Input "wildcard" has no wildcard chars, so just return tuple that will
        # cause a bucket listing to match the given input wildcard. Example: if
        # previous iteration yielded gs://bucket/dir/ with suffix_wildcard abc,
        # the next iteration will call _BuildBucketFilterStrings() with
        # gs://bucket/dir/abc, and we will return prefix ='dir/abc',
        # delimiter='/', prefix_wildcard='dir/abc', and suffix_wildcard=''.
        prefix = wildcard
        delimiter = '/'
        prefix_wildcard = wildcard
        suffix_wildcard = ''
      else:
        if match.start() > 0:
          # Wildcard does not occur at beginning of object name, so construct a
          # prefix string to send to server.
          prefix = wildcard[:match.start()]
          wildcard_part = wildcard[match.start():]
        else:
          prefix = None
          wildcard_part = wildcard
        end = wildcard_part.find('/')
        if end != -1:
          wildcard_part = wildcard_part[:end+1]
        # Remove trailing '/' so we will match gs://bucket/abc* as well as
        # gs://bucket/abc*/ with the same wildcard regex.
        prefix_wildcard = StripOneSlash((prefix or '') + wildcard_part)
        suffix_wildcard = wildcard[match.end():]
        end = suffix_wildcard.find('/')
        if end == -1:
          suffix_wildcard = ''
        else:
          suffix_wildcard = suffix_wildcard[end+1:]
        # To implement recursive (**) wildcarding, if prefix_wildcard
        # suffix_wildcard starts with '**' don't send a delimiter, and combine
        # suffix_wildcard at end of prefix_wildcard.
        if prefix_wildcard.find('**') != -1:
          delimiter = None
          prefix_wildcard += suffix_wildcard
          suffix_wildcard = ''
        else:
          delimiter = '/'
      # The following debug output is useful for tracing how the algorithm
      # walks through a multi-part wildcard like gs://bucket/abc/d*e/f*.txt
      if self.debug > 1:
        sys.stderr.write(
            'DEBUG: wildcard=%s, prefix=%s, delimiter=%s, '
            'prefix_wildcard=%s, suffix_wildcard=%s\n' %
            (wildcard, prefix, delimiter, prefix_wildcard, suffix_wildcard))
      return (prefix, delimiter, prefix_wildcard, suffix_wildcard)
  
    def _SingleVersionMatches(self, listed_generation):
      decoded_generation = GenerationFromUrlAndString(self.wildcard_url,
                                                      listed_generation)
      return str(self.wildcard_url.generation) == str(decoded_generation)
  
    def _ExpandBucketWildcards(self, bucket_fields=None):
      """Expands bucket and provider wildcards.
  
      Builds a list of bucket url strings that can be iterated on.
  
      Args:
        bucket_fields: If present, populate only these metadata fields for
                       buckets.  Example value: ['acl', 'defaultObjectAcl']
  
      Yields:
        BucketListingRefereneces of type BUCKET.
      """
      bucket_url = StorageUrlFromString(self.wildcard_url.bucket_url_string)
      if (bucket_fields and set(bucket_fields) == set(['id']) and
          not ContainsWildcard(self.wildcard_url.bucket_name)):
        # If we just want the name of a non-wildcarded bucket URL,
        # don't make an RPC.
        yield BucketListingBucket(bucket_url)
      elif(self.wildcard_url.IsBucket() and
           not ContainsWildcard(self.wildcard_url.bucket_name)):
        # If we have a non-wildcarded bucket URL, get just that bucket.
        yield BucketListingBucket(
            bucket_url, root_object=self.gsutil_api.GetBucket(
                self.wildcard_url.bucket_name, provider=self.wildcard_url.scheme,
                fields=bucket_fields))
      else:
        regex = fnmatch.translate(self.wildcard_url.bucket_name)
        prog = re.compile(regex)
  
        fields = self._GetToListFields(bucket_fields)
        if fields:
          fields.add('items/id')
        for bucket in self.gsutil_api.ListBuckets(
            fields=fields, project_id=self.project_id,
            provider=self.wildcard_url.scheme):
          if prog.match(bucket.id):
            url = StorageUrlFromString(
                '%s://%s/' % (self.wildcard_url.scheme, bucket.id))
            yield BucketListingBucket(url, root_object=bucket)
  
    def _GetToListFields(self, get_fields=None):
      """Prepends 'items/' to the input fields and converts it to a set.
  
      This way field sets requested for GetBucket can be used in ListBucket calls.
      Note that the input set must contain only bucket or object fields; listing
      fields such as prefixes or nextPageToken should be added after calling
      this function.
  
      Args:
        get_fields: Iterable fields usable in GetBucket/GetObject calls.
  
      Returns:
        Set of fields usable in ListBuckets/ListObjects calls.
      """
      if get_fields:
        list_fields = set()
        for field in get_fields:
          list_fields.add('items/' + field)
        return list_fields
  
    def _GetObjectRef(self, bucket_url_string, gcs_object, with_version=False):
      """Creates a BucketListingRef of type OBJECT from the arguments.
  
      Args:
        bucket_url_string: Wildcardless string describing the containing bucket.
        gcs_object: gsutil_api root Object for populating the BucketListingRef.
        with_version: If true, return a reference with a versioned string.
  
      Returns:
        BucketListingRef of type OBJECT.
      """
      # Generation can be None in test mocks, so just return the
      # live object for simplicity.
      if with_version and gcs_object.generation is not None:
        generation_str = GenerationFromUrlAndString(self.wildcard_url,
                                                    gcs_object.generation)
        object_string = '%s%s#%s' % (bucket_url_string, gcs_object.name,
                                     generation_str)
      else:
        object_string = '%s%s' % (bucket_url_string, gcs_object.name)
      object_url = StorageUrlFromString(object_string)
      return BucketListingObject(object_url, root_object=gcs_object)
  
    def _GetPrefixRef(self, bucket_url_string, prefix):
      """Creates a BucketListingRef of type PREFIX from the arguments.
  
      Args:
        bucket_url_string: Wildcardless string describing the containing bucket.
        prefix: gsutil_api Prefix for populating the BucketListingRef
  
      Returns:
        BucketListingRef of type PREFIX.
      """
      prefix_url = StorageUrlFromString('%s%s' % (bucket_url_string, prefix))
      return BucketListingPrefix(prefix_url, root_object=prefix)
  
    def IterBuckets(self, bucket_fields=None):
      """Iterates over the wildcard, returning refs for each expanded bucket.
  
      This ignores the object part of the URL entirely and expands only the
      the bucket portion.  It will yield BucketListingRefs of type BUCKET only.
  
      Args:
        bucket_fields: Iterable fields to include in bucket listings.
                       Ex. ['defaultObjectAcl', 'logging'].  This function is
                       responsible for converting these to listing-style
                       format ['items/defaultObjectAcl', 'items/logging'], as
                       well as adding any fields necessary for listing such as
                       'items/id'.  API implemenation is responsible for
                       adding pagination fields.  If this is None, all fields are
                       returned.
  
      Yields:
        BucketListingRef of type BUCKET, or empty iterator if no matches.
      """
      for blr in self._ExpandBucketWildcards(bucket_fields=bucket_fields):
        yield blr
  
    def IterAll(self, bucket_listing_fields=None, expand_top_level_buckets=False):
      """Iterates over the wildcard, yielding bucket, prefix or object refs.
  
      Args:
        bucket_listing_fields: If present, populate only these metadata
                               fields for listed objects.
        expand_top_level_buckets: If true and the wildcard expands only to
                                  Bucket(s), yields the expansion of each bucket
                                  into a top-level listing of prefixes and objects
                                  in that bucket instead of a BucketListingRef
                                  to that bucket.
  
      Yields:
        BucketListingRef, or empty iterator if no matches.
      """
      for blr in self.__iter__(
          bucket_listing_fields=bucket_listing_fields,
          expand_top_level_buckets=expand_top_level_buckets):
        yield blr
  
    def IterObjects(self, bucket_listing_fields=None):
      """Iterates over the wildcard, yielding only object BucketListingRefs.
  
      Args:
        bucket_listing_fields: If present, populate only these metadata
                               fields for listed objects.
  
      Yields:
        BucketListingRefs of type OBJECT or empty iterator if no matches.
      """
      for blr in self.__iter__(bucket_listing_fields=bucket_listing_fields,
                               expand_top_level_buckets=True):
        if blr.IsObject():
          yield blr
  
  
  class FileWildcardIterator(WildcardIterator):
    """WildcardIterator subclass for files and directories.
  
    If you use recursive wildcards ('**') only a single such wildcard is
    supported. For example you could use the wildcard '**/*.txt' to list all .txt
    files in any subdirectory of the current directory, but you couldn't use a
    wildcard like '**/abc/**/*.txt' (which would, if supported, let you find .txt
    files in any subdirectory named 'abc').
    """
  
    def __init__(self, wildcard_url, debug=0):
      """Instantiates an iterator over BucketListingRefs matching wildcard URL.
  
      Args:
        wildcard_url: FileUrl that contains the wildcard to iterate.
        debug: Debug level (range 0..3).
      """
      self.wildcard_url = wildcard_url
      self.debug = debug
  
    def __iter__(self):
      """Iterator that gets called when iterating over the file wildcard.
  
      In the case where no wildcard is present, returns a single matching file
      or directory.
  
      Raises:
        WildcardException: if invalid wildcard found.
  
      Yields:
        BucketListingRef of type OBJECT (for files) or PREFIX (for directories)
      """
      wildcard = self.wildcard_url.object_name
      match = FLAT_LIST_REGEX.match(wildcard)
      if match:
        # Recursive wildcarding request ('.../**/...').
        # Example input: wildcard = '/tmp/tmp2pQJAX/**/*'
        base_dir = match.group('before')[:-1]
        remaining_wildcard = match.group('after')
        # At this point for the above example base_dir = '/tmp/tmp2pQJAX' and
        # remaining_wildcard = '/*'
        if remaining_wildcard.startswith('*'):
          raise WildcardException('Invalid wildcard with more than 2 consecutive '
                                  '*s (%s)' % wildcard)
        # If there was no remaining wildcard past the recursive wildcard,
        # treat it as if it were a '*'. For example, file://tmp/** is equivalent
        # to file://tmp/**/*
        if not remaining_wildcard:
          remaining_wildcard = '*'
        # Skip slash(es).
        remaining_wildcard = remaining_wildcard.lstrip(os.sep)
        filepaths = self._IterDir(base_dir, remaining_wildcard)
      else:
        # Not a recursive wildcarding request.
        filepaths = glob.iglob(wildcard)
      for filepath in filepaths:
        expanded_url = StorageUrlFromString(filepath)
        if os.path.isdir(filepath):
          yield BucketListingPrefix(expanded_url)
        else:
          yield BucketListingObject(expanded_url)
  
    def _IterDir(self, directory, wildcard):
      """An iterator over the specified dir and wildcard."""
      # UTF8-encode directory before passing it to os.walk() so if there are
      # non-valid UTF8 chars in the file name (e.g., that can happen if the file
      # originated on Windows) os.walk() will not attempt to decode and then die
      # with a "codec can't decode byte" error, and instead we can catch the error
      # at yield time and print a more informative error message.
      for dirpath, unused_dirnames, filenames in os.walk(directory.encode(UTF8)):
        for f in fnmatch.filter(filenames, wildcard):
          try:
            yield os.path.join(dirpath, f).decode(UTF8)
          except UnicodeDecodeError:
            # Note: We considered several ways to deal with this, but each had
            # problems:
            # 1. Raise an exception and try to catch in a higher layer (the
            #    gsutil cp command), so we can properly support the gsutil cp -c
            #    option. That doesn't work because raising an exception during
            #    iteration terminates the generator.
            # 2. Accumulate a list of bad filenames and skip processing each
            #    during iteration, then raise at the end, with exception text
            #    printing the bad paths. That doesn't work because iteration is
            #    wrapped in PluralityCheckableIterator, so it's possible there
            #    are not-yet-performed copy operations at the time we reach the
            #    end of the iteration and raise the exception - which would cause
            #    us to skip copying validly named files. Moreover, the gsutil
            #    cp command loops over argv, so if you run the command gsutil cp
            #    -rc dir1 dir2 gs://bucket, an invalid unicode name inside dir1
            #    would cause dir2 never to be visited.
            # 3. Print the invalid pathname and skip it during iteration. That
            #    would work but would mean gsutil cp could exit with status 0
            #    even though some files weren't copied.
            # 4. Change the WildcardIterator to include an error status along with
            #    the result. That would solve the problem but would be a
            #    substantial change (WildcardIterator is used in many parts of
            #    gsutil), and we didn't feel that magnitude of change was
            #    warranted by this relatively uncommon corner case.
            # Instead we chose to abort when one such file is encountered, and
            # require the user to remove or rename the files and try again.
            raise CommandException('\n'.join(textwrap.wrap(
                'Invalid Unicode path encountered (%s). gsutil cannot proceed '
                'with such files present. Please remove or rename this file and '
                'try again. NOTE: the path printed above replaces the '
                'problematic characters with a hex-encoded printable '
                'representation. For more details (including how to convert to a '
                'gsutil-compatible encoding) see `gsutil help encoding`.' %
                repr(os.path.join(dirpath, f)))))
  
    # pylint: disable=unused-argument
    def IterObjects(self, bucket_listing_fields=None):
      """Iterates over the wildcard, yielding only object (file) refs.
  
      Args:
        bucket_listing_fields: Ignored as filesystems don't have buckets.
  
      Yields:
        BucketListingRefs of type OBJECT or empty iterator if no matches.
      """
      for bucket_listing_ref in self.IterAll():
        if bucket_listing_ref.IsObject():
          yield bucket_listing_ref
  
    # pylint: disable=unused-argument
    def IterAll(self, bucket_listing_fields=None, expand_top_level_buckets=False):
      """Iterates over the wildcard, yielding BucketListingRefs.
  
      Args:
        bucket_listing_fields: Ignored; filesystems don't have buckets.
        expand_top_level_buckets: Ignored; filesystems don't have buckets.
  
      Yields:
        BucketListingRefs of type OBJECT (file) or PREFIX (directory),
        or empty iterator if no matches.
      """
      for bucket_listing_ref in self.__iter__():
        yield bucket_listing_ref
  
    def IterBuckets(self, unused_bucket_fields=None):
      """Placeholder to allow polymorphic use of WildcardIterator.
  
      Args:
        unused_bucket_fields: Ignored; filesystems don't have buckets.
  
      Raises:
        WildcardException: in all cases.
      """
      raise WildcardException(
          'Iterating over Buckets not possible for file wildcards')
  
  
  class WildcardException(StandardError):
    """Exception raised for invalid wildcard URLs."""
  
    def __init__(self, reason):
      StandardError.__init__(self)
      self.reason = reason
  
    def __repr__(self):
      return 'WildcardException: %s' % self.reason
  
    def __str__(self):
      return 'WildcardException: %s' % self.reason
  
  
  def CreateWildcardIterator(url_str, gsutil_api, all_versions=False, debug=0,
                             project_id=None):
    """Instantiate a WildcardIterator for the given URL string.
  
    Args:
      url_str: URL string naming wildcard object(s) to iterate.
      gsutil_api: Cloud storage interface.  Passed in for thread safety, also
                  settable for testing/mocking.
      all_versions: If true, the iterator yields all versions of objects
                    matching the wildcard.  If false, yields just the live
                    object version.
      debug: Debug level to control debug output for iterator.
      project_id: Project id to use for bucket listings.
  
    Returns:
      A WildcardIterator that handles the requested iteration.
    """
  
    url = StorageUrlFromString(url_str)
    if url.IsFileUrl():
      return FileWildcardIterator(url, debug=debug)
    else:  # Cloud URL
      return CloudWildcardIterator(
          url, gsutil_api, all_versions=all_versions, debug=debug,
          project_id=project_id)
  