Using module 'subprocess' with timeout

后端 未结 29 2375
旧巷少年郎
旧巷少年郎 2020-11-21 15:15

Here\'s the Python code to run an arbitrary command returning its stdout data, or raise an exception on non-zero exit codes:

proc = subprocess.P         


        
相关标签:
29条回答
  • 2020-11-21 16:04

    Once you understand full process running machinery in *unix, you will easily find simplier solution:

    Consider this simple example how to make timeoutable communicate() meth using select.select() (available alsmost everythere on *nix nowadays). This also can be written with epoll/poll/kqueue, but select.select() variant could be a good example for you. And major limitations of select.select() (speed and 1024 max fds) are not applicapable for your task.

    This works under *nix, does not create threads, does not uses signals, can be lauched from any thread (not only main), and fast enought to read 250mb/s of data from stdout on my machine (i5 2.3ghz).

    There is a problem in join'ing stdout/stderr at the end of communicate. If you have huge program output this could lead to big memory usage. But you can call communicate() several times with smaller timeouts.

    class Popen(subprocess.Popen):
        def communicate(self, input=None, timeout=None):
            if timeout is None:
                return subprocess.Popen.communicate(self, input)
    
            if self.stdin:
                # Flush stdio buffer, this might block if user
                # has been writing to .stdin in an uncontrolled
                # fashion.
                self.stdin.flush()
                if not input:
                    self.stdin.close()
    
            read_set, write_set = [], []
            stdout = stderr = None
    
            if self.stdin and input:
                write_set.append(self.stdin)
            if self.stdout:
                read_set.append(self.stdout)
                stdout = []
            if self.stderr:
                read_set.append(self.stderr)
                stderr = []
    
            input_offset = 0
            deadline = time.time() + timeout
    
            while read_set or write_set:
                try:
                    rlist, wlist, xlist = select.select(read_set, write_set, [], max(0, deadline - time.time()))
                except select.error as ex:
                    if ex.args[0] == errno.EINTR:
                        continue
                    raise
    
                if not (rlist or wlist):
                    # Just break if timeout
                    # Since we do not close stdout/stderr/stdin, we can call
                    # communicate() several times reading data by smaller pieces.
                    break
    
                if self.stdin in wlist:
                    chunk = input[input_offset:input_offset + subprocess._PIPE_BUF]
                    try:
                        bytes_written = os.write(self.stdin.fileno(), chunk)
                    except OSError as ex:
                        if ex.errno == errno.EPIPE:
                            self.stdin.close()
                            write_set.remove(self.stdin)
                        else:
                            raise
                    else:
                        input_offset += bytes_written
                        if input_offset >= len(input):
                            self.stdin.close()
                            write_set.remove(self.stdin)
    
                # Read stdout / stderr by 1024 bytes
                for fn, tgt in (
                    (self.stdout, stdout),
                    (self.stderr, stderr),
                ):
                    if fn in rlist:
                        data = os.read(fn.fileno(), 1024)
                        if data == '':
                            fn.close()
                            read_set.remove(fn)
                        tgt.append(data)
    
            if stdout is not None:
                stdout = ''.join(stdout)
            if stderr is not None:
                stderr = ''.join(stderr)
    
            return (stdout, stderr)
    
    0 讨论(0)
  • 2020-11-21 16:04

    You can do this using select

    import subprocess
    from datetime import datetime
    from select import select
    
    def call_with_timeout(cmd, timeout):
        started = datetime.now()
        sp = subprocess.Popen(cmd, stdout=subprocess.PIPE)
        while True:
            p = select([sp.stdout], [], [], timeout)
            if p[0]:
                p[0][0].read()
            ret = sp.poll()
            if ret is not None:
                return ret
            if (datetime.now()-started).total_seconds() > timeout:
                sp.kill()
                return None
    
    0 讨论(0)
  • 2020-11-21 16:05

    Another option is to write to a temporary file to prevent the stdout blocking instead of needing to poll with communicate(). This worked for me where the other answers did not; for example on windows.

        outFile =  tempfile.SpooledTemporaryFile() 
        errFile =   tempfile.SpooledTemporaryFile() 
        proc = subprocess.Popen(args, stderr=errFile, stdout=outFile, universal_newlines=False)
        wait_remaining_sec = timeout
    
        while proc.poll() is None and wait_remaining_sec > 0:
            time.sleep(1)
            wait_remaining_sec -= 1
    
        if wait_remaining_sec <= 0:
            killProc(proc.pid)
            raise ProcessIncompleteError(proc, timeout)
    
        # read temp streams from start
        outFile.seek(0);
        errFile.seek(0);
        out = outFile.read()
        err = errFile.read()
        outFile.close()
        errFile.close()
    
    0 讨论(0)
  • 2020-11-21 16:05

    https://pypi.python.org/pypi/python-subprocess2 provides extensions to the subprocess module which allow you to wait up to a certain period of time, otherwise terminate.

    So, to wait up to 10 seconds for the process to terminate, otherwise kill:

    pipe  = subprocess.Popen('...')
    
    timeout =  10
    
    results = pipe.waitOrTerminate(timeout)
    

    This is compatible with both windows and unix. "results" is a dictionary, it contains "returnCode" which is the return of the app (or None if it had to be killed), as well as "actionTaken". which will be "SUBPROCESS2_PROCESS_COMPLETED" if the process completed normally, or a mask of "SUBPROCESS2_PROCESS_TERMINATED" and SUBPROCESS2_PROCESS_KILLED depending on action taken (see documentation for full details)

    0 讨论(0)
  • 2020-11-21 16:06
    import subprocess, optparse, os, sys, re, datetime, threading, time, glob, shutil, xml.dom.minidom, traceback
    
    class OutputManager:
        def __init__(self, filename, mode, console, logonly):
            self.con = console
            self.logtoconsole = True
            self.logtofile = False
    
            if filename:
                try:
                    self.f = open(filename, mode)
                    self.logtofile = True
                    if logonly == True:
                        self.logtoconsole = False
                except IOError:
                    print (sys.exc_value)
                    print ("Switching to console only output...\n")
                    self.logtofile = False
                    self.logtoconsole = True
    
        def write(self, data):
            if self.logtoconsole == True:
                self.con.write(data)
            if self.logtofile == True:
                self.f.write(data)
            sys.stdout.flush()
    
    def getTimeString():
            return time.strftime("%Y-%m-%d", time.gmtime())
    
    def runCommand(command):
        '''
        Execute a command in new thread and return the
        stdout and stderr content of it.
        '''
        try:
            Output = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True).communicate()[0]
        except Exception as e:
            print ("runCommand failed :%s" % (command))
            print (str(e))
            sys.stdout.flush()
            return None
        return Output
    
    def GetOs():
        Os = ""
        if sys.platform.startswith('win32'):
            Os = "win"
        elif sys.platform.startswith('linux'):
            Os = "linux"
        elif sys.platform.startswith('darwin'):
            Os = "mac"
        return Os
    
    
    def check_output(*popenargs, **kwargs):
        try:
            if 'stdout' in kwargs: 
                raise ValueError('stdout argument not allowed, it will be overridden.') 
    
            # Get start time.
            startTime = datetime.datetime.now()
            timeoutValue=3600
    
            cmd = popenargs[0]
    
            if sys.platform.startswith('win32'):
                process = subprocess.Popen( cmd, stdout=subprocess.PIPE, shell=True) 
            elif sys.platform.startswith('linux'):
                process = subprocess.Popen( cmd , stdout=subprocess.PIPE, shell=True ) 
            elif sys.platform.startswith('darwin'):
                process = subprocess.Popen( cmd , stdout=subprocess.PIPE, shell=True ) 
    
            stdoutdata, stderrdata = process.communicate( timeout = timeoutValue )
            retcode = process.poll()
    
            ####################################
            # Catch crash error and log it.
            ####################################
            OutputHandle = None
            try:
                if retcode >= 1:
                    OutputHandle = OutputManager( 'CrashJob_' + getTimeString() + '.txt', 'a+', sys.stdout, False)
                    OutputHandle.write( cmd )
                    print (stdoutdata)
                    print (stderrdata)
                    sys.stdout.flush()
            except Exception as e:
                print (str(e))
    
        except subprocess.TimeoutExpired:
                ####################################
                # Catch time out error and log it.
                ####################################
                Os = GetOs()
                if Os == 'win':
                    killCmd = "taskkill /FI \"IMAGENAME eq {0}\" /T /F"
                elif Os == 'linux':
                    killCmd = "pkill {0)"
                elif Os == 'mac':
                    # Linux, Mac OS
                    killCmd = "killall -KILL {0}"
    
                runCommand(killCmd.format("java"))
                runCommand(killCmd.format("YouApp"))
    
                OutputHandle = None
                try:
                    OutputHandle = OutputManager( 'KillJob_' + getTimeString() + '.txt', 'a+', sys.stdout, False)
                    OutputHandle.write( cmd )
                except Exception as e:
                    print (str(e))
        except Exception as e:
                for frame in traceback.extract_tb(sys.exc_info()[2]):
                            fname,lineno,fn,text = frame
                            print "Error in %s on line %d" % (fname, lineno)
    
    0 讨论(0)
  • 2020-11-21 16:08

    There's an idea to subclass the Popen class and extend it with some simple method decorators. Let's call it ExpirablePopen.

    from logging import error
    from subprocess import Popen
    from threading import Event
    from threading import Thread
    
    
    class ExpirablePopen(Popen):
    
        def __init__(self, *args, **kwargs):
            self.timeout = kwargs.pop('timeout', 0)
            self.timer = None
            self.done = Event()
    
            Popen.__init__(self, *args, **kwargs)
    
        def __tkill(self):
            timeout = self.timeout
            if not self.done.wait(timeout):
                error('Terminating process {} by timeout of {} secs.'.format(self.pid, timeout))
                self.kill()
    
        def expirable(func):
            def wrapper(self, *args, **kwargs):
                # zero timeout means call of parent method
                if self.timeout == 0:
                    return func(self, *args, **kwargs)
    
                # if timer is None, need to start it
                if self.timer is None:
                    self.timer = thr = Thread(target=self.__tkill)
                    thr.daemon = True
                    thr.start()
    
                result = func(self, *args, **kwargs)
                self.done.set()
    
                return result
            return wrapper
    
        wait = expirable(Popen.wait)
        communicate = expirable(Popen.communicate)
    
    
    if __name__ == '__main__':
        from subprocess import PIPE
    
        print ExpirablePopen('ssh -T git@bitbucket.org', stdout=PIPE, timeout=1).communicate()
    
    0 讨论(0)
提交回复
热议问题