# Fast large file synchronization inspired by rsync.
#
# Author: Peter Odding <peter@peterodding.com>
# Last Change: March 6, 2020
# URL: https://pdiffcopy.readthedocs.io

"""Parallel hashing of files using :mod:`multiprocessing` and :mod:`pdiffcopy.mp`."""

# Standard library modules.
import functools
import hashlib
import os

# External dependencies.
from six.moves import range

# Modules included in our package.
from pdiffcopy.mp import WorkerPool

# Public identifiers that require documentation.
__all__ = ("compute_hashes", "hash_worker")


def compute_hashes(filename, block_size, method, concurrency):
    """Compute checksums of a file in blocks (parallel)."""
    with WorkerPool(
        concurrency=concurrency,
        generator_fn=functools.partial(range, 0, os.path.getsize(filename), block_size),
        worker_fn=functools.partial(hash_worker, block_size=block_size, filename=filename, method=method),
    ) as pool:
        for offset, digest in pool:
            yield offset, digest


def hash_worker(offset, block_size, filename, method):
    """Worker function to be run in child processes."""
    with open(filename, "rb") as handle:
        handle.seek(offset)
        context = hashlib.new(method)
        context.update(handle.read(block_size))
        return offset, context.hexdigest()

[((39, 18, 39, 37), 'hashlib.new', 'hashlib.new', ({(39, 30, 39, 36): 'method'}, {}), '(method)', False, 'import hashlib\n'), ((29, 18, 29, 105), 'functools.partial', 'functools.partial', (), '', False, 'import functools\n'), ((28, 49, 28, 74), 'os.path.getsize', 'os.path.getsize', ({(28, 65, 28, 73): 'filename'}, {}), '(filename)', False, 'import os\n')]