grr/server/grr_response_server/file_store.py
def AddFilesWithUnknownHashes(
    client_path_blob_ids
):
  """Adds new files consisting of given blob ids.

  Args:
    client_path_blob_ids: A dictionary mapping `db.ClientPath` instances to
      lists of blob ids.

  Returns:
    A dictionary mapping `db.ClientPath` to hash ids of the file.

  Raises:
    BlobNotFoundError: If one of the referenced blobs cannot be found.
  """
  all_client_path_blob_ids = list()
  for client_path, blob_ids in iteritems(client_path_blob_ids):
    for blob_id in blob_ids:
      all_client_path_blob_ids.append((client_path, blob_id))

  client_path_offset = collections.defaultdict(lambda: 0)
  client_path_sha256 = collections.defaultdict(hashlib.sha256)
  client_path_blob_refs = collections.defaultdict(list)

  client_path_blob_id_batches = collection.Batch(
      items=all_client_path_blob_ids, size=_BLOBS_READ_BATCH_SIZE)

  for client_path_blob_id_batch in client_path_blob_id_batches:
    blob_id_batch = set(blob_id for _, blob_id in client_path_blob_id_batch)
    blobs = data_store.BLOBS.ReadBlobs(blob_id_batch)

    for client_path, blob_id in client_path_blob_id_batch:
      blob = blobs[blob_id]
      if blob is None:
        message = "Could not find one of referenced blobs: {}".format(blob_id)
        raise BlobNotFoundError(message)

      offset = client_path_offset[client_path]
      blob_ref = rdf_objects.BlobReference(
          offset=offset, size=len(blob), blob_id=blob_id)

      client_path_blob_refs[client_path].append(blob_ref)
      client_path_offset[client_path] = offset + len(blob)
      client_path_sha256[client_path].update(blob)

  client_path_hash_id = dict()
  hash_id_blob_refs = dict()
  for client_path in iterkeys(client_path_blob_ids):
    sha256 = client_path_sha256[client_path].digest()
    hash_id = rdf_objects.SHA256HashID.FromBytes(sha256)

    client_path_hash_id[client_path] = hash_id
    hash_id_blob_refs[hash_id] = client_path_blob_refs[client_path]

  data_store.REL_DB.WriteHashBlobReferences(hash_id_blob_refs)

  metadatas = dict()
  for client_path in iterkeys(client_path_blob_ids):
    metadatas[client_path_hash_id[client_path]] = FileMetadata(
        client_path=client_path, blob_refs=client_path_blob_refs[client_path])
  EXTERNAL_FILE_STORE.AddFiles(metadatas)

  return client_path_hash_id
Adds new files consisting of given blob ids.

  Args:
    client_path_blob_ids: A dictionary mapping `db.ClientPath` instances to
      lists of blob ids.

  Returns:
    A dictionary mapping `db.ClientPath` to hash ids of the file.

  Raises:
    BlobNotFoundError: If one of the referenced blobs cannot be found.
  
Adds new files consisting of given blob ids.
Adds new files consisting of given blob ids .
def AddFilesWithUnknownHashes(
    client_path_blob_ids
):
  all_client_path_blob_ids = list()
  for client_path, blob_ids in iteritems(client_path_blob_ids):
    for blob_id in blob_ids:
      all_client_path_blob_ids.append((client_path, blob_id))
  client_path_offset = collections.defaultdict(lambda: 0)
  client_path_sha256 = collections.defaultdict(hashlib.sha256)
  client_path_blob_refs = collections.defaultdict(list)
  client_path_blob_id_batches = collection.Batch(
      items=all_client_path_blob_ids, size=_BLOBS_READ_BATCH_SIZE)
  for client_path_blob_id_batch in client_path_blob_id_batches:
    blob_id_batch = set(blob_id for _, blob_id in client_path_blob_id_batch)
    blobs = data_store.BLOBS.ReadBlobs(blob_id_batch)
    for client_path, blob_id in client_path_blob_id_batch:
      blob = blobs[blob_id]
      if blob is None:
        message = "Could not find one of referenced blobs: {}".format(blob_id)
        raise BlobNotFoundError(message)
      offset = client_path_offset[client_path]
      blob_ref = rdf_objects.BlobReference(
          offset=offset, size=len(blob), blob_id=blob_id)
      client_path_blob_refs[client_path].append(blob_ref)
      client_path_offset[client_path] = offset + len(blob)
      client_path_sha256[client_path].update(blob)
  client_path_hash_id = dict()
  hash_id_blob_refs = dict()
  for client_path in iterkeys(client_path_blob_ids):
    sha256 = client_path_sha256[client_path].digest()
    hash_id = rdf_objects.SHA256HashID.FromBytes(sha256)
    client_path_hash_id[client_path] = hash_id
    hash_id_blob_refs[hash_id] = client_path_blob_refs[client_path]
  data_store.REL_DB.WriteHashBlobReferences(hash_id_blob_refs)
  metadatas = dict()
  for client_path in iterkeys(client_path_blob_ids):
    metadatas[client_path_hash_id[client_path]] = FileMetadata(
        client_path=client_path, blob_refs=client_path_blob_refs[client_path])
  EXTERNAL_FILE_STORE.AddFiles(metadatas)
  return client_path_hash_id
def AddFilesWithUnknownHashes ( client_path_blob_ids ) : all_client_path_blob_ids = list ( ) for client_path , blob_ids in iteritems ( client_path_blob_ids ) : for blob_id in blob_ids : all_client_path_blob_ids . append ( ( client_path , blob_id ) ) client_path_offset = collections . defaultdict ( lambda : 0 ) client_path_sha256 = collections . defaultdict ( hashlib . sha256 ) client_path_blob_refs = collections . defaultdict ( list ) client_path_blob_id_batches = collection . Batch ( items = all_client_path_blob_ids , size = _BLOBS_READ_BATCH_SIZE ) for client_path_blob_id_batch in client_path_blob_id_batches : blob_id_batch = set ( blob_id for _ , blob_id in client_path_blob_id_batch ) blobs = data_store . BLOBS . ReadBlobs ( blob_id_batch ) for client_path , blob_id in client_path_blob_id_batch : blob = blobs [ blob_id ] if blob is None : message = "Could not find one of referenced blobs: {}" . format ( blob_id ) raise BlobNotFoundError ( message ) offset = client_path_offset [ client_path ] blob_ref = rdf_objects . BlobReference ( offset = offset , size = len ( blob ) , blob_id = blob_id ) client_path_blob_refs [ client_path ] . append ( blob_ref ) client_path_offset [ client_path ] = offset + len ( blob ) client_path_sha256 [ client_path ] . update ( blob ) client_path_hash_id = dict ( ) hash_id_blob_refs = dict ( ) for client_path in iterkeys ( client_path_blob_ids ) : sha256 = client_path_sha256 [ client_path ] . digest ( ) hash_id = rdf_objects . SHA256HashID . FromBytes ( sha256 ) client_path_hash_id [ client_path ] = hash_id hash_id_blob_refs [ hash_id ] = client_path_blob_refs [ client_path ] data_store . REL_DB . WriteHashBlobReferences ( hash_id_blob_refs ) metadatas = dict ( ) for client_path in iterkeys ( client_path_blob_ids ) : metadatas [ client_path_hash_id [ client_path ] ] = FileMetadata ( client_path = client_path , blob_refs = client_path_blob_refs [ client_path ] ) EXTERNAL_FILE_STORE . AddFiles ( metadatas ) return client_path_hash_id
Adds new files consisting of given blob ids.
Adds new files consisting of given blob ids .
"""Adds new files consisting of given blob ids.

  Args:
    client_path_blob_ids: A dictionary mapping `db.ClientPath` instances to
      lists of blob ids.

  Returns:
    A dictionary mapping `db.ClientPath` to hash ids of the file.

  Raises:
    BlobNotFoundError: If one of the referenced blobs cannot be found.
  """
A dictionary mapping `db.ClientPath` to hash ids of the file.
A dictionary mapping ` db . ClientPath ` to hash ids of the file .
If one of the referenced blobs cannot be found.
If one of the referenced blobs cannot be found .
BlobNotFoundError
client_path_blob_ids
A dictionary mapping `db.ClientPath` instances to
lists of blob ids.
A dictionary mapping ` db . ClientPath ` instances to lists of blob ids .