Reccommended way to redirect file-like streams in Python?
Question: I am writing a backup script for a sqlite database that changes very
intermittently. Here's how it is now:

    
    
    from bz2 import BZ2File
    from datetime import datetime
    from os.path import dirname, abspath, join
    from hashlib import sha512
    def backup_target_database(target):
        backup_dir = dirname(abspath(target))
        hash_file = join(backup_dir, 'last_hash')
        new_hash = sha512(open(target, 'rb').read()).digest()
        if new_hash != open(hash_file, 'rb').read():
            fmt = '%Y%m%d-%H%M.sqlite3.bz2'
            snapshot_file = join(backup_dir, datetime.now().strftime(fmt))
            BZ2File(snapshot_file, 'wb').write(open(target, 'rb').read())
            open(hash_file, 'wb').write(new_hash)
    

Currently the database weighs just shy of 20MB, so it's not that taxing when
this runs and reads the whole file into memory (and do it twice when changes
are detected), but I don't want to wait until this becomes a problem.

What is the proper way to do this sort of (to use Bashscript terminology)
stream piping?


Answer: First, there's a duplication in your code (reading `target` file twice).

And you can use
[shutil.copyfileobj](https://docs.python.org/3.4/library/shutil.html#shutil.copyfileobj)
and
[hashlib.update](https://docs.python.org/3.4/library/hashlib.html#hashlib.hash.update)
for memory-efficient routine.

    
    
    from bz2 import BZ2File
    from datetime import datetime
    from hashlib import sha512
    from os.path import dirname, abspath, join
    from shutil import copyfileobj
    
    def backup_target_database(target_path):
        backup_dir = dirname(abspath(target_path))
        hash_path = join(backup_dir, 'last_hash')
        old_hash = open(hash_path, 'rb').read()
        hasher = sha512()
        with open(target_path, 'rb') as target:
            while True:
                data = target.read(1024)
                if not data:
                    break
                hasher.update(data)
            new_hash = hasher.digest()
        if new_hash != old_hash:
            fmt = '%Y%m%d-%H%M.sqlite3.bz2'
            snapshot_path = join(backup_dir, datetime.now().strftime(fmt))
            with open(target_path, 'rb') as target:
                with BZ2File(snapshot_path, 'wb', compresslevel=9) as snapshot:
                    copyfileobj(target, snapshot)
    

(Note: I didn't test this code. If you have problem please notice me)

