A non-blocking read on a subprocess.PIPE in Python

后端 未结 29 2568
醉酒成梦
醉酒成梦 2020-11-21 04:49

I\'m using the subprocess module to start a subprocess and connect to its output stream (standard output). I want to be able to execute non-blocking reads on its standard ou

相关标签:
29条回答
  • 2020-11-21 04:49

    Here is my code, used to catch every output from subprocess ASAP, including partial lines. It pumps at same time and stdout and stderr in almost correct order.

    Tested and correctly worked on Python 2.7 linux & windows.

    #!/usr/bin/python
    #
    # Runner with stdout/stderr catcher
    #
    from sys import argv
    from subprocess import Popen, PIPE
    import os, io
    from threading import Thread
    import Queue
    def __main__():
        if (len(argv) > 1) and (argv[-1] == "-sub-"):
            import time, sys
            print "Application runned!"
            time.sleep(2)
            print "Slept 2 second"
            time.sleep(1)
            print "Slept 1 additional second",
            time.sleep(2)
            sys.stderr.write("Stderr output after 5 seconds")
            print "Eol on stdin"
            sys.stderr.write("Eol on stderr\n")
            time.sleep(1)
            print "Wow, we have end of work!",
        else:
            os.environ["PYTHONUNBUFFERED"]="1"
            try:
                p = Popen( argv + ["-sub-"],
                           bufsize=0, # line-buffered
                           stdin=PIPE, stdout=PIPE, stderr=PIPE )
            except WindowsError, W:
                if W.winerror==193:
                    p = Popen( argv + ["-sub-"],
                               shell=True, # Try to run via shell
                               bufsize=0, # line-buffered
                               stdin=PIPE, stdout=PIPE, stderr=PIPE )
                else:
                    raise
            inp = Queue.Queue()
            sout = io.open(p.stdout.fileno(), 'rb', closefd=False)
            serr = io.open(p.stderr.fileno(), 'rb', closefd=False)
            def Pump(stream, category):
                queue = Queue.Queue()
                def rdr():
                    while True:
                        buf = stream.read1(8192)
                        if len(buf)>0:
                            queue.put( buf )
                        else:
                            queue.put( None )
                            return
                def clct():
                    active = True
                    while active:
                        r = queue.get()
                        try:
                            while True:
                                r1 = queue.get(timeout=0.005)
                                if r1 is None:
                                    active = False
                                    break
                                else:
                                    r += r1
                        except Queue.Empty:
                            pass
                        inp.put( (category, r) )
                for tgt in [rdr, clct]:
                    th = Thread(target=tgt)
                    th.setDaemon(True)
                    th.start()
            Pump(sout, 'stdout')
            Pump(serr, 'stderr')
    
            while p.poll() is None:
                # App still working
                try:
                    chan,line = inp.get(timeout = 1.0)
                    if chan=='stdout':
                        print "STDOUT>>", line, "<?<"
                    elif chan=='stderr':
                        print " ERROR==", line, "=?="
                except Queue.Empty:
                    pass
            print "Finish"
    
    if __name__ == '__main__':
        __main__()
    
    0 讨论(0)
  • 2020-11-21 04:49

    This is a example to run interactive command in subprocess, and the stdout is interactive by using pseudo terminal. You can refer to: https://stackoverflow.com/a/43012138/3555925

    #!/usr/bin/env python
    # -*- coding: utf-8 -*-
    
    import os
    import sys
    import select
    import termios
    import tty
    import pty
    from subprocess import Popen
    
    command = 'bash'
    # command = 'docker run -it --rm centos /bin/bash'.split()
    
    # save original tty setting then set it to raw mode
    old_tty = termios.tcgetattr(sys.stdin)
    tty.setraw(sys.stdin.fileno())
    
    # open pseudo-terminal to interact with subprocess
    master_fd, slave_fd = pty.openpty()
    
    # use os.setsid() make it run in a new process group, or bash job control will not be enabled
    p = Popen(command,
              preexec_fn=os.setsid,
              stdin=slave_fd,
              stdout=slave_fd,
              stderr=slave_fd,
              universal_newlines=True)
    
    while p.poll() is None:
        r, w, e = select.select([sys.stdin, master_fd], [], [])
        if sys.stdin in r:
            d = os.read(sys.stdin.fileno(), 10240)
            os.write(master_fd, d)
        elif master_fd in r:
            o = os.read(master_fd, 10240)
            if o:
                os.write(sys.stdout.fileno(), o)
    
    # restore tty settings back
    termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old_tty)
    
    0 讨论(0)
  • 2020-11-21 04:50

    Use select & read(1).

    import subprocess     #no new requirements
    def readAllSoFar(proc, retVal=''): 
      while (select.select([proc.stdout],[],[],0)[0]!=[]):   
        retVal+=proc.stdout.read(1)
      return retVal
    p = subprocess.Popen(['/bin/ls'], stdout=subprocess.PIPE)
    while not p.poll():
      print (readAllSoFar(p))
    

    For readline()-like:

    lines = ['']
    while not p.poll():
      lines = readAllSoFar(p, lines[-1]).split('\n')
      for a in range(len(lines)-1):
        print a
    lines = readAllSoFar(p, lines[-1]).split('\n')
    for a in range(len(lines)-1):
      print a
    
    0 讨论(0)
  • 2020-11-21 04:52

    Disclaimer: this works only for tornado

    You can do this by setting the fd to be nonblocking and then use ioloop to register callbacks. I have packaged this in an egg called tornado_subprocess and you can install it via PyPI:

    easy_install tornado_subprocess
    

    now you can do something like this:

    import tornado_subprocess
    import tornado.ioloop
    
        def print_res( status, stdout, stderr ) :
        print status, stdout, stderr
        if status == 0:
            print "OK:"
            print stdout
        else:
            print "ERROR:"
            print stderr
    
    t = tornado_subprocess.Subprocess( print_res, timeout=30, args=[ "cat", "/etc/passwd" ] )
    t.start()
    tornado.ioloop.IOLoop.instance().start()
    

    you can also use it with a RequestHandler

    class MyHandler(tornado.web.RequestHandler):
        def on_done(self, status, stdout, stderr):
            self.write( stdout )
            self.finish()
    
        @tornado.web.asynchronous
        def get(self):
            t = tornado_subprocess.Subprocess( self.on_done, timeout=30, args=[ "cat", "/etc/passwd" ] )
            t.start()
    
    0 讨论(0)
  • 2020-11-21 04:53

    Adding this answer here since it provides ability to set non-blocking pipes on Windows and Unix.

    All the ctypes details are thanks to @techtonik's answer.

    There is a slightly modified version to be used both on Unix and Windows systems.

    • Python3 compatible (only minor change needed).
    • Includes posix version, and defines exception to use for either.

    This way you can use the same function and exception for Unix and Windows code.

    # pipe_non_blocking.py (module)
    """
    Example use:
    
        p = subprocess.Popen(
                command,
                stdout=subprocess.PIPE,
                )
    
        pipe_non_blocking_set(p.stdout.fileno())
    
        try:
            data = os.read(p.stdout.fileno(), 1)
        except PortableBlockingIOError as ex:
            if not pipe_non_blocking_is_error_blocking(ex):
                raise ex
    """
    
    
    __all__ = (
        "pipe_non_blocking_set",
        "pipe_non_blocking_is_error_blocking",
        "PortableBlockingIOError",
        )
    
    import os
    
    
    if os.name == "nt":
        def pipe_non_blocking_set(fd):
            # Constant could define globally but avoid polluting the name-space
            # thanks to: https://stackoverflow.com/questions/34504970
            import msvcrt
    
            from ctypes import windll, byref, wintypes, WinError, POINTER
            from ctypes.wintypes import HANDLE, DWORD, BOOL
    
            LPDWORD = POINTER(DWORD)
    
            PIPE_NOWAIT = wintypes.DWORD(0x00000001)
    
            def pipe_no_wait(pipefd):
                SetNamedPipeHandleState = windll.kernel32.SetNamedPipeHandleState
                SetNamedPipeHandleState.argtypes = [HANDLE, LPDWORD, LPDWORD, LPDWORD]
                SetNamedPipeHandleState.restype = BOOL
    
                h = msvcrt.get_osfhandle(pipefd)
    
                res = windll.kernel32.SetNamedPipeHandleState(h, byref(PIPE_NOWAIT), None, None)
                if res == 0:
                    print(WinError())
                    return False
                return True
    
            return pipe_no_wait(fd)
    
        def pipe_non_blocking_is_error_blocking(ex):
            if not isinstance(ex, PortableBlockingIOError):
                return False
            from ctypes import GetLastError
            ERROR_NO_DATA = 232
    
            return (GetLastError() == ERROR_NO_DATA)
    
        PortableBlockingIOError = OSError
    else:
        def pipe_non_blocking_set(fd):
            import fcntl
            fl = fcntl.fcntl(fd, fcntl.F_GETFL)
            fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
            return True
    
        def pipe_non_blocking_is_error_blocking(ex):
            if not isinstance(ex, PortableBlockingIOError):
                return False
            return True
    
        PortableBlockingIOError = BlockingIOError
    

    To avoid reading incomplete data, I ended up writing my own readline generator (which returns the byte string for each line).

    Its a generator so you can for example...

    def non_blocking_readlines(f, chunk=1024):
        """
        Iterate over lines, yielding b'' when nothings left
        or when new data is not yet available.
    
        stdout_iter = iter(non_blocking_readlines(process.stdout))
    
        line = next(stdout_iter)  # will be a line or b''.
        """
        import os
    
        from .pipe_non_blocking import (
                pipe_non_blocking_set,
                pipe_non_blocking_is_error_blocking,
                PortableBlockingIOError,
                )
    
        fd = f.fileno()
        pipe_non_blocking_set(fd)
    
        blocks = []
    
        while True:
            try:
                data = os.read(fd, chunk)
                if not data:
                    # case were reading finishes with no trailing newline
                    yield b''.join(blocks)
                    blocks.clear()
            except PortableBlockingIOError as ex:
                if not pipe_non_blocking_is_error_blocking(ex):
                    raise ex
    
                yield b''
                continue
    
            while True:
                n = data.find(b'\n')
                if n == -1:
                    break
    
                yield b''.join(blocks) + data[:n + 1]
                data = data[n + 1:]
                blocks.clear()
            blocks.append(data)
    
    0 讨论(0)
  • 2020-11-21 04:54

    Things are a lot better in modern Python.

    Here's a simple child program, "hello.py":

    #!/usr/bin/env python3
    
    while True:
        i = input()
        if i == "quit":
            break
        print(f"hello {i}")
    

    And a program to interact with it:

    import asyncio
    
    
    async def main():
        proc = await asyncio.subprocess.create_subprocess_exec(
            "./hello.py", stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE
        )
        proc.stdin.write(b"bob\n")
        print(await proc.stdout.read(1024))
        proc.stdin.write(b"alice\n")
        print(await proc.stdout.read(1024))
        proc.stdin.write(b"quit\n")
        await proc.wait()
    
    
    asyncio.run(main())
    

    That prints out:

    b'hello bob\n'
    b'hello alice\n'
    

    Note that the actual pattern, which is also by almost all of the previous answers, both here and in related questions, is to set the child's stdout file descriptor to non-blocking and then poll it in some sort of select loop. These days, of course, that loop is provided by asyncio.

    0 讨论(0)
提交回复
热议问题