I've got a long-running process. Once it finishes I need the output. But the user should be informed while it's running. With the logging module I get timestamps and whatnot preceding. So let's combine all 3:Prelude:import osimport loggingimport subprocessfrom io import IOBasefrom sys import stdoutfrom select import selectfrom threading import Threadfrom time import sleepfrom cStringIO import StringIOInit:# Some other file, like __init__.pylogging.basicConfig(format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',                    level='INFO')handler = logging.root.handlers.pop()assert logging.root.handlers == [], root logging handlers aren't emptyhandler.stream.close()handler.stream = stdoutlogging.root.addHandler(handler)# Some other file, like __init__.pylog = logging.getLogger(__name__)Class (based on https://stackoverflow.com/a/4838875):class StreamLogger(IOBase, logging.Handler):    _run = None    def __init__(self, logger_obj, level):        super(StreamLogger, self).__init__()        self.logger_obj = logger_obj        self.level = level        self.pipe = os.pipe()        self.thread = Thread(target=self._flusher)        self.thread.start()    def __call__(self):        return self    def _flusher(self):        self._run = True        buf = b''        while self._run:            for fh in select([self.pipe[0]], [], [], 1)[0]:                buf += os.read(fh, 1024)                while b'\n' in buf:                    data, buf = buf.split(b'\n', 1)                    self.write(data.decode())        self._run = None    def write(self, data):        return self.logger_obj.log(self.level, data)    emit = write    def fileno(self):        return self.pipe[1]    def close(self):        if self._run:            self._run = False            while self._run is not None:                sleep(1)            for pipe in self.pipe:                os.close(pipe)            self.thread.join(1)Filtering class (based on https://stackoverflow.com/a/41328542):class LevelRangeFilter(logging.Filter):    def __init__(self, min_level, max_level, name=''):        super(LevelRangeFilter, self).__init__(name)        self._min_level = min_level        self._max_level = max_level    def filter(self, record):        return super(LevelRangeFilter, self).filter(record) and (            self._min_level is None or self._min_level <= record.levelno) and (                   self._max_level is None or record.levelno < self._max_level)Usage:stderr_stream = logging.StreamHandler(StringIO())stderr_stream.addFilter(LevelRangeFilter(logging.ERROR, None))log.addHandler(stderr_stream)log.setLevel(logging.ERROR)stdout_stream = logging.StreamHandler(StringIO())stdout_stream.addFilter(LevelRangeFilter(logging.INFO, logging.ERROR))log.addHandler(stdout_stream)log.setLevel(logging.INFO)with StreamLogger(log, logging.INFO) as out, StreamLogger(log,                                                          logging.ERROR) as err:    subprocess.Popen('ls', stdout=out, stderr=err, shell=True)print 'stderr_tee =', stderr_tee.stream.getvalue()print 'stdout_tee =', stdout_tee.stream.getvalue()With a cleanup like so:# finally blockfor handler in log.handlers:    log.removeHandler(handler)    handler.stream.close()    handler.close()stderr_tee.stream.close()stdout_tee.stream.close()
subprocess with logging module: tee with StringIO
python;python 2.7;io;logging;child process