'Check all queued up hashes for existence in file store.
Hashes which do not exist in the file store will be downloaded. This
function flushes the entire queue (self.state.pending_hashes) in order to
minimize the round trips to the file store.
If a file was found in the file store it is copied from there into the
client\'s VFS namespace. Otherwise, we request the client to hash every block
in the file, and add it to the file tracking queue
(self.state.pending_files).'

def _CheckHashesWithFileStore(self):

    if (not self.state.pending_hashes):
        return
    file_hashes = {}
    hash_to_tracker = {}
    for (index, tracker) in self.state.pending_hashes.iteritems():
        if (tracker.get('hash_obj') is None):
            continue
        hash_obj = tracker['hash_obj']
        digest = hash_obj.sha256
        file_hashes[index] = hash_obj
        hash_to_tracker.setdefault(digest, []).append(tracker)
    files_in_filestore = {}
    filestore_obj = aff4.FACTORY.Open(filestore.FileStore.PATH, filestore.FileStore, mode='r', token=self.token)
    for (file_store_urn, hash_obj) in filestore_obj.CheckHashes(file_hashes.values(), external=self.state.use_external_stores):
        self.HeartBeat()
        for tracker in hash_to_tracker[hash_obj.sha256]:
            self.state.files_skipped += 1
            file_hashes.pop(tracker['index'])
            files_in_filestore[file_store_urn] = hash_obj
            self.state.pending_hashes.pop(tracker['index'])
    self.state.files_hashed_since_check = 0
    for (filestore_file_urn, hash_obj) in files_in_filestore.iteritems():
        for file_tracker in hash_to_tracker.get(hash_obj.sha256, []):
            stat_entry = file_tracker['stat_entry']
            target_urn = stat_entry.pathspec.AFF4Path(self.client_id)
            aff4.FACTORY.Copy(filestore_file_urn, target_urn, update_timestamps=True, sync=True, token=self.token)
            with aff4.FACTORY.Open(target_urn, mode='rw', token=self.token) as new_fd:
                new_fd.Set(new_fd.Schema.STAT, stat_entry)
                if (new_fd.size == 0):
                    new_fd.size = (file_tracker['bytes_read'] or stat_entry.st_size)
            filestore_obj.AddURNToIndex(str(hash_obj.sha256), target_urn)
            self._ReceiveFetchedFile(file_tracker)
    for index in file_hashes:
        file_tracker = self.state.pending_hashes.pop(index)
        self.state.pending_files[index] = file_tracker
        if (file_tracker['bytes_read'] > 0):
            file_tracker['size_to_download'] = file_tracker['bytes_read']
        else:
            file_tracker['size_to_download'] = file_tracker['stat_entry'].st_size
        expected_number_of_hashes = ((file_tracker['size_to_download'] / self.CHUNK_SIZE) + 1)
        self.state.files_to_fetch += 1
        for i in range(expected_number_of_hashes):
            if (i == (expected_number_of_hashes - 1)):
                length = (file_tracker['size_to_download'] % self.CHUNK_SIZE)
            else:
                length = self.CHUNK_SIZE
            self.CallClient(server_stubs.HashBuffer, pathspec=file_tracker['stat_entry'].pathspec, offset=(i * self.CHUNK_SIZE), length=length, next_state='CheckHash', request_data=dict(index=index))
    if ((self.state.files_hashed % 100) == 0):
        self.Log('Hashed    %d    files,    skipped    %s    already    stored.', self.state.files_hashed, self.state.files_skipped)
