src/dirbs/importer/importer_utils.py
def split_file(input_file, lines, output_dir, logger, output_file_basename='split_file'):
    """Method to split an input file into multiple files and return the split filenames."""
    batch_filename_base = os.path.join(output_dir, output_file_basename)
    batch_line_count = 0
    batch_num = 0

    # Read the header from the file
    header = input_file.readline()

    if not header.rstrip():
        # If file is empty, just return a new empty file to allow importers that accept empty files to work
        filename = '{0}.{1:d}'.format(batch_filename_base, batch_num)
        with open(filename, 'wb'):
            pass
        yield filename
        # Indicate that we are done with this generated
        return

    buf = io.BytesIO()
    buf.write(header)
    for line in input_file:
        buf.write(line)
        batch_line_count += 1

        # If we're starting a new batch, open a file
        if batch_line_count % lines == 0:  # noqa: S001
            filename = '{0}.{1:d}'.format(batch_filename_base, batch_num)
            write_buffer_to_file(buf, filename)
            buf = io.BytesIO()
            buf.write(header)
            batch_num += 1
            batch_line_count = 0
            if batch_num > 0 and batch_num % 50 == 0:
                logger.info('Written {0} split files'.format(batch_num))

            # Yield filename back
            yield filename

    # Handle final batch. We need to check if line count is > 0 (ie. there is content to write). If we never
    # wrote a batch we still need to make sure we emit the header so we have an or check for batch_num == 0
    if batch_line_count > 0 or batch_num == 0:
        filename = '{0}.{1:d}'.format(batch_filename_base, batch_num)
        write_buffer_to_file(buf, filename)
        yield filename

    # Return to indicate that we are done
    return
Method to split an input file into multiple files and return the split filenames.
Method to split an input file into multiple files and return the split filenames.
Method to split an input file into multiple files and return the split filenames .
def split_file(input_file, lines, output_dir, logger, output_file_basename='split_file'):
    batch_filename_base = os.path.join(output_dir, output_file_basename)
    batch_line_count = 0
    batch_num = 0
    header = input_file.readline()
    if not header.rstrip():
        filename = '{0}.{1:d}'.format(batch_filename_base, batch_num)
        with open(filename, 'wb'):
            pass
        yield filename
        return
    buf = io.BytesIO()
    buf.write(header)
    for line in input_file:
        buf.write(line)
        batch_line_count += 1
        if batch_line_count % lines == 0:  
            filename = '{0}.{1:d}'.format(batch_filename_base, batch_num)
            write_buffer_to_file(buf, filename)
            buf = io.BytesIO()
            buf.write(header)
            batch_num += 1
            batch_line_count = 0
            if batch_num > 0 and batch_num % 50 == 0:
                logger.info('Written {0} split files'.format(batch_num))
            yield filename
    if batch_line_count > 0 or batch_num == 0:
        filename = '{0}.{1:d}'.format(batch_filename_base, batch_num)
        write_buffer_to_file(buf, filename)
        yield filename
    return
def split_file ( input_file , lines , output_dir , logger , output_file_basename = 'split_file' ) : batch_filename_base = os . path . join ( output_dir , output_file_basename ) batch_line_count = 0 batch_num = 0 header = input_file . readline ( ) if not header . rstrip ( ) : filename = '{0}.{1:d}' . format ( batch_filename_base , batch_num ) with open ( filename , 'wb' ) : pass yield filename return buf = io . BytesIO ( ) buf . write ( header ) for line in input_file : buf . write ( line ) batch_line_count += 1 if batch_line_count % lines == 0 : filename = '{0}.{1:d}' . format ( batch_filename_base , batch_num ) write_buffer_to_file ( buf , filename ) buf = io . BytesIO ( ) buf . write ( header ) batch_num += 1 batch_line_count = 0 if batch_num > 0 and batch_num % 50 == 0 : logger . info ( 'Written {0} split files' . format ( batch_num ) ) yield filename if batch_line_count > 0 or batch_num == 0 : filename = '{0}.{1:d}' . format ( batch_filename_base , batch_num ) write_buffer_to_file ( buf , filename ) yield filename return
Method to split an input file into multiple files and return the split filenames.
Method to split an input file into multiple files and return the split filenames .
"""Method to split an input file into multiple files and return the split filenames.""" # Read the header from the file # If file is empty, just return a new empty file to allow importers that accept empty files to work # Indicate that we are done with this generated # If we're starting a new batch, open a file # noqa: S001 # Yield filename back # Handle final batch. We need to check if line count is > 0 (ie. there is content to write). If we never # wrote a batch we still need to make sure we emit the header so we have an or check for batch_num == 0 # Return to indicate that we are done
input_file
lines
output_dir
logger
output_file_basename