grr/server/grr_response_server/gui/archive_generator_aff4.py
def Generate(self, items, token=None):
    """Generates archive from a given collection.

    Iterates the collection and generates an archive by yielding contents
    of every referenced AFF4Stream.

    Args:
      items: Iterable with items that point to aff4 paths.
      token: User's ACLToken.

    Yields:
      Binary chunks comprising the generated archive.
    """
    clients = set()
    for fd_urn_batch in collection.Batch(
        self._ItemsToUrns(items), self.BATCH_SIZE):
      self.total_files += len(fd_urn_batch)

      fds_to_write = {}
      for fd in aff4.FACTORY.MultiOpen(fd_urn_batch, token=token):
        # Derive a ClientPath from AFF4 URN to make new and old
        # archive_generator predicate input consistent.
        # TODO(user): This code is clearly hacky and intended to be removed.
        urn_components = fd.urn.Split()
        if urn_components[1:3] != ["fs", "os"]:
          raise AssertionError("URN components are expected to start with "
                               "client, 'fs', 'os'. Got %r" % (urn_components,))

        client_path = db.ClientPath.OS(
            client_id=urn_components[0], components=urn_components[3:])

        if not self.predicate(client_path):
          self.ignored_files.add(utils.SmartUnicode(fd.urn))
          continue

        # Any file-like object with data in AFF4 should inherit AFF4Stream.
        if isinstance(fd, aff4.AFF4Stream):
          urn_components = fd.urn.Split()
          clients.add(rdf_client.ClientURN(urn_components[0]))

          content_path = os.path.join(self.prefix, *urn_components)

          # Make sure size of the original file is passed. It's required
          # when output_writer is StreamingTarWriter.
          st = os.stat_result((0o644, 0, 0, 0, 0, 0, fd.size, 0, 0, 0))
          fds_to_write[fd] = (content_path, st)

      if fds_to_write:
        prev_fd = None
        for fd, chunk, exception in aff4.AFF4Stream.MultiStream(fds_to_write):
          if exception:
            logging.exception(exception)

            try:
              self.archived_files.remove(utils.SmartUnicode(fd.urn))
            except KeyError:
              pass  # Failing is fine, since removal should be idempotent.
            self.failed_files.add(utils.SmartUnicode(fd.urn))
            continue

          if prev_fd != fd:
            if prev_fd:
              yield self.archive_generator.WriteFileFooter()
            prev_fd = fd

            content_path, st = fds_to_write[fd]
            yield self.archive_generator.WriteFileHeader(content_path, st=st)

          yield self.archive_generator.WriteFileChunk(chunk)
          self.archived_files.add(utils.SmartUnicode(fd.urn))

        if self.archive_generator.is_file_write_in_progress:
          yield self.archive_generator.WriteFileFooter()

    if clients:
      for client_urn_batch in collection.Batch(clients, self.BATCH_SIZE):
        for fd in aff4.FACTORY.MultiOpen(
            client_urn_batch, aff4_type=aff4_grr.VFSGRRClient, token=token):
          for chunk in self._GenerateClientInfo(fd):
            yield chunk

    for chunk in self._GenerateDescription():
      yield chunk

    yield self.archive_generator.Close()
Generates archive from a given collection.

    Iterates the collection and generates an archive by yielding contents
    of every referenced AFF4Stream.

    Args:
      items: Iterable with items that point to aff4 paths.
      token: User's ACLToken.

    Yields:
      Binary chunks comprising the generated archive.
    
Generates archive from a given collection.
Iterates the collection and generates an archive by yielding contents
of every referenced AFF4Stream.
Generates archive from a given collection . Iterates the collection and generates an archive by yielding contents of every referenced AFF4Stream .
def Generate(self, items, token=None):
    clients = set()
    for fd_urn_batch in collection.Batch(
        self._ItemsToUrns(items), self.BATCH_SIZE):
      self.total_files += len(fd_urn_batch)
      fds_to_write = {}
      for fd in aff4.FACTORY.MultiOpen(fd_urn_batch, token=token):
        urn_components = fd.urn.Split()
        if urn_components[1:3] != ["fs", "os"]:
          raise AssertionError("URN components are expected to start with "
                               "client, 'fs', 'os'. Got %r" % (urn_components,))
        client_path = db.ClientPath.OS(
            client_id=urn_components[0], components=urn_components[3:])
        if not self.predicate(client_path):
          self.ignored_files.add(utils.SmartUnicode(fd.urn))
          continue
        if isinstance(fd, aff4.AFF4Stream):
          urn_components = fd.urn.Split()
          clients.add(rdf_client.ClientURN(urn_components[0]))
          content_path = os.path.join(self.prefix, *urn_components)
          st = os.stat_result((0o644, 0, 0, 0, 0, 0, fd.size, 0, 0, 0))
          fds_to_write[fd] = (content_path, st)
      if fds_to_write:
        prev_fd = None
        for fd, chunk, exception in aff4.AFF4Stream.MultiStream(fds_to_write):
          if exception:
            logging.exception(exception)
            try:
              self.archived_files.remove(utils.SmartUnicode(fd.urn))
            except KeyError:
              pass  
            self.failed_files.add(utils.SmartUnicode(fd.urn))
            continue
          if prev_fd != fd:
            if prev_fd:
              yield self.archive_generator.WriteFileFooter()
            prev_fd = fd
            content_path, st = fds_to_write[fd]
            yield self.archive_generator.WriteFileHeader(content_path, st=st)
          yield self.archive_generator.WriteFileChunk(chunk)
          self.archived_files.add(utils.SmartUnicode(fd.urn))
        if self.archive_generator.is_file_write_in_progress:
          yield self.archive_generator.WriteFileFooter()
    if clients:
      for client_urn_batch in collection.Batch(clients, self.BATCH_SIZE):
        for fd in aff4.FACTORY.MultiOpen(
            client_urn_batch, aff4_type=aff4_grr.VFSGRRClient, token=token):
          for chunk in self._GenerateClientInfo(fd):
            yield chunk
    for chunk in self._GenerateDescription():
      yield chunk
    yield self.archive_generator.Close()
def Generate ( self , items , token = None ) : clients = set ( ) for fd_urn_batch in collection . Batch ( self . _ItemsToUrns ( items ) , self . BATCH_SIZE ) : self . total_files += len ( fd_urn_batch ) fds_to_write = { } for fd in aff4 . FACTORY . MultiOpen ( fd_urn_batch , token = token ) : urn_components = fd . urn . Split ( ) if urn_components [ 1 : 3 ] != [ "fs" , "os" ] : raise AssertionError ( "URN components are expected to start with " "client, 'fs', 'os'. Got %r" % ( urn_components , ) ) client_path = db . ClientPath . OS ( client_id = urn_components [ 0 ] , components = urn_components [ 3 : ] ) if not self . predicate ( client_path ) : self . ignored_files . add ( utils . SmartUnicode ( fd . urn ) ) continue if isinstance ( fd , aff4 . AFF4Stream ) : urn_components = fd . urn . Split ( ) clients . add ( rdf_client . ClientURN ( urn_components [ 0 ] ) ) content_path = os . path . join ( self . prefix , * urn_components ) st = os . stat_result ( ( 0o644 , 0 , 0 , 0 , 0 , 0 , fd . size , 0 , 0 , 0 ) ) fds_to_write [ fd ] = ( content_path , st ) if fds_to_write : prev_fd = None for fd , chunk , exception in aff4 . AFF4Stream . MultiStream ( fds_to_write ) : if exception : logging . exception ( exception ) try : self . archived_files . remove ( utils . SmartUnicode ( fd . urn ) ) except KeyError : pass self . failed_files . add ( utils . SmartUnicode ( fd . urn ) ) continue if prev_fd != fd : if prev_fd : yield self . archive_generator . WriteFileFooter ( ) prev_fd = fd content_path , st = fds_to_write [ fd ] yield self . archive_generator . WriteFileHeader ( content_path , st = st ) yield self . archive_generator . WriteFileChunk ( chunk ) self . archived_files . add ( utils . SmartUnicode ( fd . urn ) ) if self . archive_generator . is_file_write_in_progress : yield self . archive_generator . WriteFileFooter ( ) if clients : for client_urn_batch in collection . Batch ( clients , self . BATCH_SIZE ) : for fd in aff4 . FACTORY . MultiOpen ( client_urn_batch , aff4_type = aff4_grr . VFSGRRClient , token = token ) : for chunk in self . _GenerateClientInfo ( fd ) : yield chunk for chunk in self . _GenerateDescription ( ) : yield chunk yield self . archive_generator . Close ( )
Generates archive from a given collection.
Generates archive from a given collection .
"""Generates archive from a given collection.

    Iterates the collection and generates an archive by yielding contents
    of every referenced AFF4Stream.

    Args:
      items: Iterable with items that point to aff4 paths.
      token: User's ACLToken.

    Yields:
      Binary chunks comprising the generated archive.
    """ # Derive a ClientPath from AFF4 URN to make new and old # archive_generator predicate input consistent. # TODO(user): This code is clearly hacky and intended to be removed. # Any file-like object with data in AFF4 should inherit AFF4Stream. # Make sure size of the original file is passed. It's required # when output_writer is StreamingTarWriter. # Failing is fine, since removal should be idempotent.
Binary chunks comprising the generated archive.
Binary chunks comprising the generated archive .
self
items
Iterable with items that point to aff4 paths.
Iterable with items that point to aff4 paths .
token
None