Unbuffered Python Subrocess PIPE

问题

I have been trying to implement a wrapper around subprocess as follows:

def ans_cmd_stream_color(inputcmd):
"""Driver function for local ansible commands.

Stream stdout to stdout and log file with color.
Runs <inputcmd> via subprocess.
Returns return code, stdout, stderr as dict.
"""
fullcmd = inputcmd
create_debug('Enabling colorful ansible output.', LOGGER)
create_info('Running command: ' + fullcmd, LOGGER, True)
p = subprocess.Popen('export ANSIBLE_FORCE_COLOR=true; ' + fullcmd,
                     stdout=subprocess.PIPE,
                     stderr=subprocess.PIPE,
                     shell=True)
stdout_l = []
stderr_l = []
rcode = 0
# Regex black magic
ansi_escape = re.compile(r'\x1b[^m]*m')
# Get the unbuffered IO action going.
try:
    # Non blocking
    reads = [p.stdout.fileno(), p.stderr.fileno()]
    ret = select.select(reads, [], [])
    # Print line by line
    while True:
        for fd in ret[0]:
            if fd == p.stdout.fileno():
                line = p.stdout.readline()
                sys.stdout.write(line.encode('utf-8'))
                stdout_l.append(ansi_escape.sub('',
                                                line.encode('utf-8'))
                                )
            if fd == p.stderr.fileno():
                line = p.stdout.readline()
                sys.stderr.write(line.encode('utf-8'))
                stderr_l.append(ansi_escape.sub('',
                                                line.encode('utf-8'))
                                )
        # Break when the process is done.
        if p.poll() is not None:
            rcode = p.returncode
            break
except BaseException as e:
    raise e
outstr = ''.join(stdout_l)
errstr = ''.join(stderr_l)
outstr, errstr = str(outstr).rstrip('\n'), str(errstr).rstrip('\n')
expstr = errstr.strip('ERROR: ')
if len(expstr) >= 1:
    create_info('Command: ' + str(fullcmd) + ': ' + expstr + '\n', LOGGER,
                True)
    if rcode == 0:
        rcode = 1
else:
    create_info(outstr + '\n', LOGGER)
    if rcode == 0:
        create_info('Command: ' + fullcmd + ' ran successfully.', LOGGER,
                    True)
    expstr = False
ret_dict = {inputcmd: {}}
ret_dict[inputcmd]['rcode'] = rcode
ret_dict[inputcmd]['stdout'] = outstr
ret_dict[inputcmd]['stderr'] = expstr
return copy.deepcopy(ret_dict)

The idea is to print a streaming output of subprocess command and then return info to the function user. The issue is that even using a direct io.open, the subprocess PIP is still buffered unless I set:

os.environ["PYTHONUNBUFFERED"] = "1"

Which is not ideal. Any ideas or has anybody encountered this issue?

UPDATE: With ansible you need to disable buffering for subprocess to honor buffering settings:

def ans_cmd_stream_color(inputcmd):
"""Driver function for local ansible commands.

Stream stdout to stdout and log file with color.
Runs <inputcmd> via subprocess.
Returns return code, stdout, stderr as dict.
"""
fullcmd = inputcmd
create_debug('Enabling colorful ansible output.', LOGGER)
create_info('Running command: ' + fullcmd, LOGGER, True)
p = subprocess.Popen('export ANSIBLE_FORCE_COLOR=true; ' +
                     'export PYTHONUNBUFFERED=1; ' + fullcmd,
                     stdout=subprocess.PIPE,
                     stderr=subprocess.PIPE,
                     shell=True)
stdout_l = []
stderr_l = []
rcode = 0
# Regex black magic
ansi_escape = re.compile(r'\x1b[^m]*m')
# Get the unbuffered IO action going.
try:
    # Non blocking
    reads = [p.stdout.fileno(), p.stderr.fileno()]
    ret = select.select(reads, [], [])
    # Print line by line
    while True:
        for fd in ret[0]:
            if fd == p.stdout.fileno():
                line = p.stdout.readline()
                sys.stdout.write(line.encode('utf-8'))
                stdout_l.append(ansi_escape.sub('',
                                                line.encode('utf-8'))
                                )
            if fd == p.stderr.fileno():
                line = p.stdout.readline()
                sys.stderr.write(line.encode('utf-8'))
                stderr_l.append(ansi_escape.sub('',
                                                line.encode('utf-8'))
                                )
        # Break when the process is done.
        if p.poll() is not None:
            rcode = p.returncode
            break
except BaseException as e:
    raise e
outstr = ''.join(stdout_l)
errstr = ''.join(stderr_l)
outstr, errstr = str(outstr).rstrip('\n'), str(errstr).rstrip('\n')
expstr = errstr.strip('ERROR: ')
if len(expstr) >= 1:
    create_info('Command: ' + str(fullcmd) + ': ' + expstr + '\n', LOGGER,
                True)
    if rcode == 0:
        rcode = 1
else:
    create_info(outstr + '\n', LOGGER)
    if rcode == 0:
        create_info('Command: ' + fullcmd + ' ran successfully.', LOGGER,
                    True)
    expstr = False
ret_dict = {inputcmd: {}}
ret_dict[inputcmd]['rcode'] = rcode
ret_dict[inputcmd]['stdout'] = outstr
ret_dict[inputcmd]['stderr'] = expstr
return copy.deepcopy(ret_dict)

回答1:

You should probably directly read from the subprocess pipes. Something like the following will read from the standard out to the information logger and the standard error to the error logger.

import logging, subprocess
logging.basicConfig(level=logging.INFO)
proc = subprocess.Popen(
        cmd, stdout=subprocess.PIPE,
        stderr=subprocess.PIPE
    )
cont = True
while cont:
    cont = False
    line = proc.stdout.readline()
    if not line == b"":
        out = line.decode("utf-8").rstrip()
        logging.info(out)
        cont = True

    line = proc.stderr.readline()
    if not line == b"":
        out = line.decode("utf-8").rstrip()
        logging.error(out)
        cont = True

    if not cont and proc.poll() is not None:
        break

---

To address the buffering issue, per this question, either the subordinate Python script must explicitly flush the buffers, or the environment variable PYTHONUNBUFFERED must be set to a non-empty string.

来源：https://stackoverflow.com/questions/37123805/unbuffered-python-subrocess-pipe