examples/event-pipeline-demo/event_pipeline_demo/pipelines.py
def gunzipper(gzip_file):
    '''gunzips /path/to/foo.gz to /path/to/raw/2019/01/01/data.json
    '''
    # TODO: take date as an input

    path_prefix = os.path.dirname(gzip_file)
    output_folder = os.path.join(path_prefix, 'raw/2019/01/01')
    outfile = os.path.join(output_folder, 'data.json')

    if not safe_isfile(outfile):
        mkdir_p(output_folder)

        with gzip.open(gzip_file, 'rb') as f_in, open(outfile, 'wb') as f_out:
            shutil.copyfileobj(f_in, f_out)

    return [path_prefix]
def gunzipper(gzip_file):
    '''gunzips /path/to/foo.gz to /path/to/raw/2019/01/01/data.json
    '''
    # TODO: take date as an input

    path_prefix = os.path.dirname(gzip_file)
    output_folder = os.path.join(path_prefix, 'raw/2019/01/01')
    outfile = os.path.join(output_folder, 'data.json')

    if not safe_isfile(outfile):
        mkdir_p(output_folder)

        with gzip.open(gzip_file, 'rb') as f_in, open(outfile, 'wb') as f_out:
            shutil.copyfileobj(f_in, f_out)

    return [path_prefix]
def gunzipper ( gzip_file ) : # TODO: take date as an input path_prefix = os . path . dirname ( gzip_file ) output_folder = os . path . join ( path_prefix , 'raw/2019/01/01' ) outfile = os . path . join ( output_folder , 'data.json' ) if not safe_isfile ( outfile ) : mkdir_p ( output_folder ) with gzip . open ( gzip_file , 'rb' ) as f_in , open ( outfile , 'wb' ) as f_out : shutil . copyfileobj ( f_in , f_out ) return [ path_prefix ]
gunzips /path/to/foo.gz to /path/to/raw/2019/01/01/data.json
gunzips / path / to / foo . gz to / path / to / raw / 2019 / 01 / 01 / data . json