clusterjob/utils.py
def run_cmd(cmd, remote, rootdir='', workdir='', ignore_exit_code=False,
        ssh='ssh'):
    r'''Run the given cmd in the given workdir, either locally or remotely, and
    return the combined stdout/stderr

    Parameters:
        cmd (list of str or str): Command to execute, as list consisting of the
            command, and options.  Alternatively, the command can be given a
            single string, which will then be executed as a shell command. Only
            use shell commands when necessary, e.g. when the command involves a
            pipe.
        remote (None or str): If None, run command locally. Otherwise, run on
            the given host (via SSH)
        rootdir (str, optional): Local or remote root directory. The `workdir`
            variable is taken relative to `rootdir`. If not specified,
            effectively the current working directory is used as the root for
            local commands, and the home directory for remote commands. Note
            that `~` may be used to indicate the home directory locally or
            remotely.
        workdir (str, optional): Local or remote directory from which to run
            the command, relative to `rootdir`. If `rootdir` is empty, `~` may
            be used to indicate the home directory.
        ignore_exit_code (boolean, optional): By default,
            `subprocess.CalledProcessError` will be raised if the call has an
            exit code other than 0. This exception can be supressed by passing
            `ignore_exit_code=False`
        ssh (str, optional): The executable to be used for ssh. If not a full
            path, the executable must be in ``$PATH``

    Example:

        >>> import tempfile, os, shutil
        >>> tempfolder = tempfile.mkdtemp()
        >>> scriptfile = os.path.join(tempfolder, 'test.sh')
        >>> with open(scriptfile, 'w') as script_fh:
        ...     script_fh.writelines(["#!/bin/bash\n", "echo Hello $1\n"])
        >>> set_executable(scriptfile)

        >>> run_cmd(['./test.sh', 'World'], remote=None, workdir=tempfolder)
        'Hello World\n'

        >>> run_cmd("./test.sh World | tr '[:upper:]' '[:lower:]'", remote=None,
        ...         workdir=tempfolder)
        'hello world\n'

        >>> shutil.rmtree(tempfolder)
    '''
    logger = logging.getLogger(__name__)
    workdir = os.path.join(rootdir, workdir)
    if type(cmd) in [list, tuple]:
        use_shell = False
    else:
        cmd = str(cmd)
        use_shell = True
    try:
        if remote is None: # run locally
            workdir = os.path.expanduser(workdir)
            if use_shell:
                logger.debug("COMMAND: %s", cmd)
            else:
                logger.debug("COMMAND: %s",
                             " ".join([quote(part) for part in cmd]))
            if workdir == '':
                response = sp.check_output(cmd, stderr=sp.STDOUT,
                                           shell=use_shell)
            else:
                response = sp.check_output(cmd, stderr=sp.STDOUT, cwd=workdir,
                                           shell=use_shell)
        else: # run remotely
            if not use_shell:
                cmd = " ".join(cmd)
            if workdir == '':
                cmd = [ssh, remote, cmd]
            else:
                cmd = [ssh, remote, 'cd %s && %s' % (workdir, cmd)]
            logger.debug("COMMAND: %s",
                         " ".join([quote(part) for part in cmd]))
            response = sp.check_output(cmd, stderr=sp.STDOUT)
    except sp.CalledProcessError as e:
        if ignore_exit_code:
            response = e.output
        else:
            raise
    if sys.version_info >= (3, 0):
        # For Python 3, we should return a unicode string, so that the backends
        # can safely assume that string operations such as regex matching are
        # possible.
        response = response.decode(CMD_RESPONSE_ENCODING)
    if logger.getEffectiveLevel() <= logging.DEBUG:
        if "\n" in response:
            if len(response.splitlines()) == 1:
                logger.debug("RESPONSE: %s", response)
            else:
                logger.debug("RESPONSE: ---\n%s\n---", response)
        else:
            logger.debug("RESPONSE: '%s'", response)
    return response
r'''Run the given cmd in the given workdir, either locally or remotely, and
    return the combined stdout/stderr

    Parameters:
        cmd (list of str or str): Command to execute, as list consisting of the
            command, and options.  Alternatively, the command can be given a
            single string, which will then be executed as a shell command. Only
            use shell commands when necessary, e.g. when the command involves a
            pipe.
        remote (None or str): If None, run command locally. Otherwise, run on
            the given host (via SSH)
        rootdir (str, optional): Local or remote root directory. The `workdir`
            variable is taken relative to `rootdir`. If not specified,
            effectively the current working directory is used as the root for
            local commands, and the home directory for remote commands. Note
            that `~` may be used to indicate the home directory locally or
            remotely.
        workdir (str, optional): Local or remote directory from which to run
            the command, relative to `rootdir`. If `rootdir` is empty, `~` may
            be used to indicate the home directory.
        ignore_exit_code (boolean, optional): By default,
            `subprocess.CalledProcessError` will be raised if the call has an
            exit code other than 0. This exception can be supressed by passing
            `ignore_exit_code=False`
        ssh (str, optional): The executable to be used for ssh. If not a full
            path, the executable must be in ``$PATH``

    Example:

        >>> import tempfile, os, shutil
        >>> tempfolder = tempfile.mkdtemp()
        >>> scriptfile = os.path.join(tempfolder, 'test.sh')
        >>> with open(scriptfile, 'w') as script_fh:
        ...     script_fh.writelines(["#!/bin/bash\n", "echo Hello $1\n"])
        >>> set_executable(scriptfile)

        >>> run_cmd(['./test.sh', 'World'], remote=None, workdir=tempfolder)
        'Hello World\n'

        >>> run_cmd("./test.sh World | tr '[:upper:]' '[:lower:]'", remote=None,
        ...         workdir=tempfolder)
        'hello world\n'

        >>> shutil.rmtree(tempfolder)
    
r'''Run the given cmd in the given workdir, either locally or remotely, and
return the combined stdout/stderr
r ' ' ' Run the given cmd in the given workdir either locally or remotely and return the combined stdout / stderr
def run_cmd(cmd, remote, rootdir='', workdir='', ignore_exit_code=False,
        ssh='ssh'):
    logger = logging.getLogger(__name__)
    workdir = os.path.join(rootdir, workdir)
    if type(cmd) in [list, tuple]:
        use_shell = False
    else:
        cmd = str(cmd)
        use_shell = True
    try:
        if remote is None: 
            workdir = os.path.expanduser(workdir)
            if use_shell:
                logger.debug("COMMAND: %s", cmd)
            else:
                logger.debug("COMMAND: %s",
                             " ".join([quote(part) for part in cmd]))
            if workdir == '':
                response = sp.check_output(cmd, stderr=sp.STDOUT,
                                           shell=use_shell)
            else:
                response = sp.check_output(cmd, stderr=sp.STDOUT, cwd=workdir,
                                           shell=use_shell)
        else: 
            if not use_shell:
                cmd = " ".join(cmd)
            if workdir == '':
                cmd = [ssh, remote, cmd]
            else:
                cmd = [ssh, remote, 'cd %s && %s' % (workdir, cmd)]
            logger.debug("COMMAND: %s",
                         " ".join([quote(part) for part in cmd]))
            response = sp.check_output(cmd, stderr=sp.STDOUT)
    except sp.CalledProcessError as e:
        if ignore_exit_code:
            response = e.output
        else:
            raise
    if sys.version_info >= (3, 0):
        response = response.decode(CMD_RESPONSE_ENCODING)
    if logger.getEffectiveLevel() <= logging.DEBUG:
        if "\n" in response:
            if len(response.splitlines()) == 1:
                logger.debug("RESPONSE: %s", response)
            else:
                logger.debug("RESPONSE: ---\n%s\n---", response)
        else:
            logger.debug("RESPONSE: '%s'", response)
    return response
def run_cmd ( cmd , remote , rootdir = '' , workdir = '' , ignore_exit_code = False , ssh = 'ssh' ) : logger = logging . getLogger ( __name__ ) workdir = os . path . join ( rootdir , workdir ) if type ( cmd ) in [ list , tuple ] : use_shell = False else : cmd = str ( cmd ) use_shell = True try : if remote is None : workdir = os . path . expanduser ( workdir ) if use_shell : logger . debug ( "COMMAND: %s" , cmd ) else : logger . debug ( "COMMAND: %s" , " " . join ( [ quote ( part ) for part in cmd ] ) ) if workdir == '' : response = sp . check_output ( cmd , stderr = sp . STDOUT , shell = use_shell ) else : response = sp . check_output ( cmd , stderr = sp . STDOUT , cwd = workdir , shell = use_shell ) else : if not use_shell : cmd = " " . join ( cmd ) if workdir == '' : cmd = [ ssh , remote , cmd ] else : cmd = [ ssh , remote , 'cd %s && %s' % ( workdir , cmd ) ] logger . debug ( "COMMAND: %s" , " " . join ( [ quote ( part ) for part in cmd ] ) ) response = sp . check_output ( cmd , stderr = sp . STDOUT ) except sp . CalledProcessError as e : if ignore_exit_code : response = e . output else : raise if sys . version_info >= ( 3 , 0 ) : response = response . decode ( CMD_RESPONSE_ENCODING ) if logger . getEffectiveLevel ( ) <= logging . DEBUG : if "\n" in response : if len ( response . splitlines ( ) ) == 1 : logger . debug ( "RESPONSE: %s" , response ) else : logger . debug ( "RESPONSE: ---\n%s\n---" , response ) else : logger . debug ( "RESPONSE: '%s'" , response ) return response
r'''Run the given cmd in the given workdir, either locally or remotely, and
return the combined stdout/stderr
r ' ' ' Run the given cmd in the given workdir either locally or remotely and return the combined stdout / stderr
r'''Run the given cmd in the given workdir, either locally or remotely, and
    return the combined stdout/stderr

    Parameters:
        cmd (list of str or str): Command to execute, as list consisting of the
            command, and options.  Alternatively, the command can be given a
            single string, which will then be executed as a shell command. Only
            use shell commands when necessary, e.g. when the command involves a
            pipe.
        remote (None or str): If None, run command locally. Otherwise, run on
            the given host (via SSH)
        rootdir (str, optional): Local or remote root directory. The `workdir`
            variable is taken relative to `rootdir`. If not specified,
            effectively the current working directory is used as the root for
            local commands, and the home directory for remote commands. Note
            that `~` may be used to indicate the home directory locally or
            remotely.
        workdir (str, optional): Local or remote directory from which to run
            the command, relative to `rootdir`. If `rootdir` is empty, `~` may
            be used to indicate the home directory.
        ignore_exit_code (boolean, optional): By default,
            `subprocess.CalledProcessError` will be raised if the call has an
            exit code other than 0. This exception can be supressed by passing
            `ignore_exit_code=False`
        ssh (str, optional): The executable to be used for ssh. If not a full
            path, the executable must be in ``$PATH``

    Example:

        >>> import tempfile, os, shutil
        >>> tempfolder = tempfile.mkdtemp()
        >>> scriptfile = os.path.join(tempfolder, 'test.sh')
        >>> with open(scriptfile, 'w') as script_fh:
        ...     script_fh.writelines(["#!/bin/bash\n", "echo Hello $1\n"])
        >>> set_executable(scriptfile)

        >>> run_cmd(['./test.sh', 'World'], remote=None, workdir=tempfolder)
        'Hello World\n'

        >>> run_cmd("./test.sh World | tr '[:upper:]' '[:lower:]'", remote=None,
        ...         workdir=tempfolder)
        'hello world\n'

        >>> shutil.rmtree(tempfolder)
    ''' # run locally # run remotely # For Python 3, we should return a unicode string, so that the backends # can safely assume that string operations such as regex matching are # possible.
cmd
Command to execute, as list consisting of the
command, and options.  Alternatively, the command can be given a
single string, which will then be executed as a shell command. Only
use shell commands when necessary, e.g. when the command involves a
pipe.
Command to execute as list consisting of the command and options . Alternatively the command can be given a single string which will then be executed as a shell command . Only use shell commands when necessary e . g . when the command involves a pipe .
remote
If None, run command locally. Otherwise, run on
the given host (via SSH)
If None run command locally . Otherwise run on the given host ( via SSH )
rootdir
Local or remote root directory. The `workdir`
variable is taken relative to `rootdir`. If not specified,
effectively the current working directory is used as the root for
local commands, and the home directory for remote commands. Note
that `~` may be used to indicate the home directory locally or
remotely.
Local or remote root directory . The ` workdir ` variable is taken relative to ` rootdir ` . If not specified effectively the current working directory is used as the root for local commands and the home directory for remote commands . Note that ` ~ ` may be used to indicate the home directory locally or remotely .
workdir
Local or remote directory from which to run
the command, relative to `rootdir`. If `rootdir` is empty, `~` may
be used to indicate the home directory.
Local or remote directory from which to run the command relative to ` rootdir ` . If ` rootdir ` is empty ` ~ ` may be used to indicate the home directory .
ignore_exit_code
By default,
`subprocess.CalledProcessError` will be raised if the call has an
exit code other than 0. This exception can be supressed by passing
`ignore_exit_code=False`
By default ` subprocess . CalledProcessError ` will be raised if the call has an exit code other than 0 . This exception can be supressed by passing ` ignore_exit_code = False `
ssh
The executable to be used for ssh. If not a full
path, the executable must be in ``$PATH``
The executable to be used for ssh . If not a full path the executable must be in ` ` $PATH ` `
examples
>>> import tempfile, os, shutil
>>> tempfolder = tempfile.mkdtemp()
>>> scriptfile = os.path.join(tempfolder, 'test.sh')
>>> with open(scriptfile, 'w') as script_fh:
script_fh.writelines(["#!/bin/bash\n", "echo Hello $1\n"])
>>> set_executable(scriptfile)

>>> run_cmd(['./test.sh', 'World'], remote=None, workdir=tempfolder)
'Hello World\n'

>>> run_cmd("./test.sh World | tr '[:upper:]' '[:lower:]'", remote=None,
workdir=tempfolder)
'hello world\n'

>>> shutil.rmtree(tempfolder)
>>> import tempfile os shutil >>> tempfolder = tempfile . mkdtemp () >>> scriptfile = os . path . join ( tempfolder ' test . sh ' ) >>> with open ( scriptfile ' w ' ) as script_fh : script_fh . writelines ( [ " #! / bin / bash \ n " " echo Hello $1 \ n " ] ) >>> set_executable ( scriptfile ) >>> run_cmd ( [ ' . / test . sh ' ' World ' ] remote = None workdir = tempfolder ) ' Hello World \ n ' >>> run_cmd ( " . / test . sh World | tr ' [ : upper : ] ' ' [ : lower : ] ' " remote = None workdir = tempfolder ) ' hello world \ n ' >>> shutil . rmtree ( tempfolder )