How to gzip the result while writing in Python?
Question: I am parsing some code and writing the result into 11 text files but the
problem is that my ssd cannot afford the normal sizes . that is why I am
looking for a way to write the results in a gzipped format.any idea?

my code:

    
    
    from __future__ import print_function, division
    import os
    from glob import glob
    from os import getpid
    from time import time
    from sys import stdout
    import resource
    from multiprocessing import Pool
    import subprocess
    try:
        import ujson as json
    except ImportError:
        try:
            import simplejson as json
        except ImportError:
            import json
    
    
    path = '/data/data/A.1/unzipped/*.A.1'
    print("Running with PID: %d" % getpid())
    
    
    
    def process_file(file):
        start = time()
        filename =file.split('/')[-1]
        print(file)
        with open('/data/data/A.1/%s_DI' %filename, 'w') as w:
            with open(file, 'r') as f:
                for n, line in enumerate(f):
                    d = json.loads(line)
                    try:
    
                        domain = d['rrname']
                        ips = d['rdata']
                        for i in ips:
                            print("%s|%s" % (i, domain), file=w)
                    except:
                        pass
    
    
    if __name__ == "__main__":
        files_list = glob(path)
        cores = 12
        print("Using %d cores" % cores)
        pp = Pool(processes=cores)
        pp.imap_unordered(process_file, files_list)
        pp.close()
        pp.join()
    


Answer: 
    import logging
    logging.basicConfig(level=logging.DEBUG, filename='logfile')
    # ....
    with gzip.GzipFile('/data/data/A.1/{0}_DI.gz'.format(filename), 'wb') as w:
        logging.debug('{} Opened for writing'.format(filename))
        with open(file, 'r') as f:
            logging.debug('{} opened for reading'.format(file))
            for n, line in enumerate(f):
                d = json.loads(line)
                logging.debug('{0} <=== json loaded from {1}'.format(d, line))
                try:
                    domain = d['rrname']
                    logging.debug('domain: {}'.format(domain))
                    ips = d['rdata']
                    logging.debug('ips: {}'.format(ips))
                    for i in ips:
                        print("%s|%s" % (i, domain), file=w)
                except:
                    pass
    

Hope that helps...

