Append to compressed tar file with performance As you might know, Python's tarfile does not have the ability to append to the files that compressed with e.g. gz or bz2. I tried to implement this functionality which works fine but operates slowly. The following function accepts a string or bytes object and appends it as a file to an existing tarfile. I'm not sure if this code has the best performance, and it may have issues. I also tried writing files to memory instead of to a temporary directory, but this didn't impact performance.
import os
import tarfile
import tempfile
import time
from pathlib import Path


def append_tar_file(buffer, file_name, output_path, replace=True):
    &quot;&quot;&quot;
    append a buffer to an existing tar file
    &quot;&quot;&quot;
    # extract files
    # check for existing file and overwrite if need to
    # compress files
    if not os.path.isfile(output_path):
        return
    buffer = buffer.encode(&quot;utf-8&quot;) if isinstance(buffer, str) else buffer

    with tempfile.TemporaryDirectory() as tempdir:
        tempdirp = Path(tempdir)
        with tarfile.open(output_path, &quot;r:bz2&quot;) as tar:
            try:
                tar.extractall(os.path.abspath(tempdirp))
            except Exception as err:   #tar file is empty
                print(err)
        buffer_path = os.path.join(tempdir, os.path.basename(file_name))
        if replace or (buffer_path not in list(os.path.abspath(f) for f in tempdirp.iterdir())):
            with open(buffer_path, &quot;wb&quot;) as f:
                f.write(buffer)

        with tarfile.open(output_path, &quot;w:bz2&quot;) as tar:
            for file in tempdirp.iterdir():
                try:
                    tar.add(file, arcname=os.path.basename(os.path.normpath(file)))
                except Exception as err:
                    print(err)

if __name__ == &quot;__main__&quot;:
    path = &quot;./test.tar.gz&quot;
    buffer = &quot;Test String&quot;
    filename = &quot;somefile&quot;
    for i in range(1, 100):
        print(time.time())
        append_tar_file(buffer, filename+str(i), path)
Indeed, the tarfile package doesn't support appending to a compressed tar.
But I think you can do better than your current attempt.
Instead of extracting the content to disk, you could keep it in memory, write and append to a new compressed file, and finally rename the compressed file.
That way you will write a single temporary file,
instead of many.
The reduced disk I/O should improve the performance.

def append_tar_file(buf, file_name, output_path, replace=True):
    """
    append a buf to an existing tar file if not already there, or if replace=True
    """
    if not os.path.isfile(output_path):
        return

    with tempfile.TemporaryDirectory() as tempdir:
        tmp_path = os.path.join(tempdir, 'tmp.tar.bz2')

        with tarfile.open(output_path, "r:bz2") as tar:
            if not replace:
                if file_name in (member.name for member in tar):
                    return

            if isinstance(buf, str):
                buf = buf.encode("utf-8")

            fileobj = BytesIO(buf)
            tarinfo = tarfile.TarInfo(file_name)
            tarinfo.size = len(fileobj.getvalue())

            with tarfile.open(tmp_path, "w:bz2") as tmp:
                for member in tar:
                    if member.name != file_name:
                        tmp.addfile(member, tar.extractfile(member.name))
                tmp.addfile(tarinfo, fileobj)

        os.rename(tmp_path, output_path)