grr/server/grr_response_server/file_store.py
def StreamFilesChunks(client_paths, max_timestamp=None, max_size=None):
  """Streams contents of given files.

  Args:
    client_paths: db.ClientPath objects describing paths to files.
    max_timestamp: If specified, then for every requested file will open the
      last collected version of the file with a timestamp equal or lower than
      max_timestamp. If not specified, will simply open a latest version for
      each file.
    max_size: If specified, only the chunks covering max_size bytes will be
      returned.

  Yields:
    StreamedFileChunk objects for every file read. Chunks will be returned
    sequentially, their order will correspond to the client_paths order.
    Files having no content will simply be ignored.
  """

  path_infos_by_cp = (
      data_store.REL_DB.ReadLatestPathInfosWithHashBlobReferences(
          client_paths, max_timestamp=max_timestamp))

  hash_ids_by_cp = {
      cp: rdf_objects.SHA256HashID.FromBytes(pi.hash_entry.sha256.AsBytes())
      for cp, pi in iteritems(path_infos_by_cp)
      if pi
  }

  blob_refs_by_hash_id = data_store.REL_DB.ReadHashBlobReferences(
      hash_ids_by_cp.values())

  all_chunks = []
  for cp in client_paths:
    try:
      hash_id = hash_ids_by_cp[cp]
    except KeyError:
      continue

    try:
      blob_refs = blob_refs_by_hash_id[hash_id]
    except KeyError:
      continue

    num_blobs = len(blob_refs)
    total_size = 0
    for ref in blob_refs:
      total_size += ref.size

    cur_size = 0
    for i, ref in enumerate(blob_refs):
      all_chunks.append((cp, ref.blob_id, i, num_blobs, ref.offset, total_size))

      cur_size += ref.size
      if max_size is not None and cur_size >= max_size:
        break

  for batch in collection.Batch(all_chunks, STREAM_CHUNKS_READ_AHEAD):
    blobs = data_store.BLOBS.ReadBlobs(
        [blob_id for cp, blob_id, i, num_blobs, offset, total_size in batch])
    for cp, blob_id, i, num_blobs, offset, total_size in batch:
      yield StreamedFileChunk(cp, blobs[blob_id], i, num_blobs, offset,
                              total_size)
Streams contents of given files.

  Args:
    client_paths: db.ClientPath objects describing paths to files.
    max_timestamp: If specified, then for every requested file will open the
      last collected version of the file with a timestamp equal or lower than
      max_timestamp. If not specified, will simply open a latest version for
      each file.
    max_size: If specified, only the chunks covering max_size bytes will be
      returned.

  Yields:
    StreamedFileChunk objects for every file read. Chunks will be returned
    sequentially, their order will correspond to the client_paths order.
    Files having no content will simply be ignored.
  
Streams contents of given files.
Streams contents of given files .
def StreamFilesChunks(client_paths, max_timestamp=None, max_size=None):
  path_infos_by_cp = (
      data_store.REL_DB.ReadLatestPathInfosWithHashBlobReferences(
          client_paths, max_timestamp=max_timestamp))
  hash_ids_by_cp = {
      cp: rdf_objects.SHA256HashID.FromBytes(pi.hash_entry.sha256.AsBytes())
      for cp, pi in iteritems(path_infos_by_cp)
      if pi
  }
  blob_refs_by_hash_id = data_store.REL_DB.ReadHashBlobReferences(
      hash_ids_by_cp.values())
  all_chunks = []
  for cp in client_paths:
    try:
      hash_id = hash_ids_by_cp[cp]
    except KeyError:
      continue
    try:
      blob_refs = blob_refs_by_hash_id[hash_id]
    except KeyError:
      continue
    num_blobs = len(blob_refs)
    total_size = 0
    for ref in blob_refs:
      total_size += ref.size
    cur_size = 0
    for i, ref in enumerate(blob_refs):
      all_chunks.append((cp, ref.blob_id, i, num_blobs, ref.offset, total_size))
      cur_size += ref.size
      if max_size is not None and cur_size >= max_size:
        break
  for batch in collection.Batch(all_chunks, STREAM_CHUNKS_READ_AHEAD):
    blobs = data_store.BLOBS.ReadBlobs(
        [blob_id for cp, blob_id, i, num_blobs, offset, total_size in batch])
    for cp, blob_id, i, num_blobs, offset, total_size in batch:
      yield StreamedFileChunk(cp, blobs[blob_id], i, num_blobs, offset,
                              total_size)
def StreamFilesChunks ( client_paths , max_timestamp = None , max_size = None ) : path_infos_by_cp = ( data_store . REL_DB . ReadLatestPathInfosWithHashBlobReferences ( client_paths , max_timestamp = max_timestamp ) ) hash_ids_by_cp = { cp : rdf_objects . SHA256HashID . FromBytes ( pi . hash_entry . sha256 . AsBytes ( ) ) for cp , pi in iteritems ( path_infos_by_cp ) if pi } blob_refs_by_hash_id = data_store . REL_DB . ReadHashBlobReferences ( hash_ids_by_cp . values ( ) ) all_chunks = [ ] for cp in client_paths : try : hash_id = hash_ids_by_cp [ cp ] except KeyError : continue try : blob_refs = blob_refs_by_hash_id [ hash_id ] except KeyError : continue num_blobs = len ( blob_refs ) total_size = 0 for ref in blob_refs : total_size += ref . size cur_size = 0 for i , ref in enumerate ( blob_refs ) : all_chunks . append ( ( cp , ref . blob_id , i , num_blobs , ref . offset , total_size ) ) cur_size += ref . size if max_size is not None and cur_size >= max_size : break for batch in collection . Batch ( all_chunks , STREAM_CHUNKS_READ_AHEAD ) : blobs = data_store . BLOBS . ReadBlobs ( [ blob_id for cp , blob_id , i , num_blobs , offset , total_size in batch ] ) for cp , blob_id , i , num_blobs , offset , total_size in batch : yield StreamedFileChunk ( cp , blobs [ blob_id ] , i , num_blobs , offset , total_size )
Streams contents of given files.
Streams contents of given files .
"""Streams contents of given files.

  Args:
    client_paths: db.ClientPath objects describing paths to files.
    max_timestamp: If specified, then for every requested file will open the
      last collected version of the file with a timestamp equal or lower than
      max_timestamp. If not specified, will simply open a latest version for
      each file.
    max_size: If specified, only the chunks covering max_size bytes will be
      returned.

  Yields:
    StreamedFileChunk objects for every file read. Chunks will be returned
    sequentially, their order will correspond to the client_paths order.
    Files having no content will simply be ignored.
  """
StreamedFileChunk objects for every file read. Chunks will be returned
sequentially, their order will correspond to the client_paths order.
Files having no content will simply be ignored.
StreamedFileChunk objects for every file read . Chunks will be returned sequentially their order will correspond to the client_paths order . Files having no content will simply be ignored .
client_paths
db.ClientPath objects describing paths to files.
db . ClientPath objects describing paths to files .
max_timestamp
If specified, then for every requested file will open the
last collected version of the file with a timestamp equal or lower than
max_timestamp. If not specified, will simply open a latest version for
each file.
If specified then for every requested file will open the last collected version of the file with a timestamp equal or lower than max_timestamp . If not specified will simply open a latest version for each file .
max_size
If specified, only the chunks covering max_size bytes will be
returned.
If specified only the chunks covering max_size bytes will be returned .