pyroSAR/gamma/parser.py
def parse_command(command):
    """
    Parse the help text of a Gamma command to a Python function including a docstring.
    The docstring is in rst format and can thu be parsed by e.g. sphinx.
    This function is not intended to be used by itself, but rather within function :func:`parse_module`.

    Parameters
    ----------
    command: str
        the name of the gamma command

    Returns
    -------
    str
        the full Python function text

    """
    command = which(command)
    proc = sp.Popen(command, stdin=sp.PIPE, stdout=sp.PIPE, stderr=sp.PIPE, universal_newlines=True)
    out, err = proc.communicate()
    out += err
    
    # filter header command description and usage description text
    header = '\n'.join([x.strip('* ') for x in re.findall('[*]{3}.*[*]{3}', out)])
    header = '| ' + header.replace('\n', '\n| ')
    usage = re.search('usage:.*(?=\n)', out).group()
    
    # filter required and optional arguments from usage description text
    arg_req = [re.sub('[^\w.-]*', '', x) for x in re.findall('[^<]*<([^>]*)>', usage)]
    arg_opt = [re.sub('[^\w.-]*', '', x) for x in re.findall('[^[]*\[([^]]*)\]', usage)]
    
    # fix inconsistencies in parameter naming related to case differences,
    # e.g. ISP_PAR in the usage text vs. ISP_Par in the parameter description
    for arg in arg_req + arg_opt:
        for item in re.findall(arg, out, re.IGNORECASE):
            if item != arg:
                out = out.replace(item, arg)
    
    double = [k for k, v in Counter(arg_req + arg_opt).items() if v > 1]
    if len(double) > 0:
        raise RuntimeError('double parameter{0}: {1}'.format('s' if len(double)> 1 else '', ', '.join(double)))
    
    # print('header_raw: \n{}\n'.format(header))
    # print('usage_raw: \n{}\n'.format(usage))
    # print('required args: {}\n'.format(', '.join(arg_req)))
    # print('optional args: {}\n'.format(', '.join(arg_opt)))
    # print('double args: {}\n'.format(', '.join(double)))
    
    # create the function argument string for the Python function
    # optional arguments are parametrized with '-' as default value, e.g. arg_opt='-'
    # a '-' in the parameter name is replaced with '_'
    # example: "arg1, arg2, arg3='-'"
    argstr_function = re.sub(r'([^\'])-([^\'])', r'\1_\2', ', '.join(arg_req + [x + "='-'" for x in arg_opt])) \
        .replace(', def=', ', drm=')
    
    # create the process call argument string
    # a '-' in the parameter name is replaced with '_'
    # e.g. 'arg1, arg2, arg3'
    # if a parameter is named 'def' (not allowed in Python) it is renamed to 'drm'
    argstr_process = ', '.join(arg_req + arg_opt) \
        .replace('-', '_') \
        .replace(', def,', ', drm,')
    
    # print('arg_str1: \n{}\n'.format(argstr_function))
    # print('arg_str2: \n{}\n'.format(argstr_process))
    
    # define the start of the parameter documentation string, which is either after 'input_parameters' or after
    # the usage description string
    doc_start = 'input parameters:[ ]*\n' if re.search('input parameters', out) else 'usage:.*(?=\n)'
    
    # parse the parameter documentation to a Python docstring format
    
    # define the number of spaces to indent
    indent = ' ' * 4
    
    docstring_elements = ['Parameters\n----------']
    
    # gather the indices, which mark the documentation start of the respective parameters within 
    # the raw documentation text
    starts = []
    for x in arg_req + arg_opt:
        try:
            starts.append(re.search(r'\n[ ]*{0} .*'.format(x), out).start())
        except AttributeError:
            raise RuntimeError('cannot find parameter {}'.format(x))
    starts += [len(out)]
    
    # define a pattern for parsing individual parameter documentations
    pattern = r'\n[ ]*(?P<par>{0})[ ]+(?P<doc>.*)'.format('|'.join(arg_req + arg_opt))
    # print(pattern)
    
    for i in range(0, len(starts) - 1):
        # draw a subset from the Gamma docstring containing only the doc of a single parameter
        doc_raw = out[starts[i]:starts[i + 1]]
        # print(repr(doc_raw))
        
        # parse the docstring
        match = re.match(pattern, doc_raw, flags=re.DOTALL)
        if not match:
            continue
        
        # retrieve the parameter name and the documentation lines
        par = match.group('par')
        doc_items = re.split('\n+\s*', match.group('doc').strip('\n'))
        
        # escape * characters (which are treated as special characters for bullet lists by sphinx)
        doc_items = [x.replace('*', '\*') for x in doc_items]
        
        # convert all lines starting with an integer number or 'NOTE' to bullet list items
        latest = None
        for i in range(len(doc_items)):
            item = doc_items[i]
            if re.search('^(?:(?:-|)[-0-9]+|NOTE):', item):
                latest = i
                # prepend '* ' and replace missing spaces after a colon: 'x:x' -> 'x: x'
                doc_items[i] = '* ' + re.sub(r'((?:-|)[-0-9]+:)(\w+)', r'\1 \2', item)
        
        # format documentation lines coming after the last bullet list item
        # sphinx expects lines after the last bullet item to be indented by two spaces if
        # they belong to the bullet item or otherwise a blank line to mark the end of the bullet list
        if latest:
            # case if there are still lines coming after the last bullet item,
            # prepend an extra two spaces to these lines so that they are properly
            # aligned with the text of the bullet item
            if latest + 2 <= len(doc_items):
                i = 1
                while latest + i + 1 <= len(doc_items):
                    doc_items[latest + i] = '  ' + doc_items[latest + i]
                    i += 1
            # if not, then insert an extra blank line
            else:
                doc_items[-1] = doc_items[-1] + '\n'
        
        # parse the final documentation string for the current parameter
        description = '\n{0}{0}'.join(doc_items).format(indent)
        doc = '{0}:\n{1}{2}'.format(par, indent, description)
        docstring_elements.append(doc)
    
    # create docstring for parameter logpath
    doc = 'logpath: str or None\n{0}a directory to write command logfiles to'.format(indent)
    docstring_elements.append(doc)
    
    # create the function definition string
    fun_def = 'def {name}({args_fun}, logpath=None):' \
        .format(name=os.path.basename(command).replace('-', '_'),
                args_fun=argstr_function)
    
    # create the complete docstring
    fun_doc = '\n{header}\n\n{doc}\n' \
        .format(header=header,
                doc='\n'.join(docstring_elements))
    
    # create the process call string
    fun_proc = "process(['{command}', {args_cmd}], logpath=logpath)" \
        .format(command=command,
                args_cmd=argstr_process)
    
    # combine the elements to a complete Python function string
    fun = '''{defn}\n"""{doc}"""\n{proc}'''.format(defn=fun_def, doc=fun_doc, proc=fun_proc)
    
    # indent all lines and add an extra empty line at the end
    fun = fun.replace('\n', '\n{}'.format(indent)) + '\n'
    
    return fun

    Parse the help text of a Gamma command to a Python function including a docstring.
    The docstring is in rst format and can thu be parsed by e.g. sphinx.
    This function is not intended to be used by itself, but rather within function :func:`parse_module`.

    Parameters
    ----------
    command: str
        the name of the gamma command

    Returns
    -------
    str
        the full Python function text

    
Parse the help text of a Gamma command to a Python function including a docstring.
The docstring is in rst format and can thu be parsed by e.g. sphinx.
This function is not intended to be used by itself, but rather within function :func:`parse_module`.

Parameters

str
the name of the gamma command

Returns

str
the full Python function text
Parse the help text of a Gamma command to a Python function including a docstring . The docstring is in rst format and can thu be parsed by e . g . sphinx . This function is not intended to be used by itself but rather within function : func : ` parse_module ` . Parameters str the name of the gamma command Returns str the full Python function text
def parse_command(command):
    command = which(command)
    proc = sp.Popen(command, stdin=sp.PIPE, stdout=sp.PIPE, stderr=sp.PIPE, universal_newlines=True)
    out, err = proc.communicate()
    out += err
    header = '\n'.join([x.strip('* ') for x in re.findall('[*]{3}.*[*]{3}', out)])
    header = '| ' + header.replace('\n', '\n| ')
    usage = re.search('usage:.*(?=\n)', out).group()
    arg_req = [re.sub('[^\w.-]*', '', x) for x in re.findall('[^<]*<([^>]*)>', usage)]
    arg_opt = [re.sub('[^\w.-]*', '', x) for x in re.findall('[^[]*\[([^]]*)\]', usage)]
    for arg in arg_req + arg_opt:
        for item in re.findall(arg, out, re.IGNORECASE):
            if item != arg:
                out = out.replace(item, arg)
    double = [k for k, v in Counter(arg_req + arg_opt).items() if v > 1]
    if len(double) > 0:
        raise RuntimeError('double parameter{0}: {1}'.format('s' if len(double)> 1 else '', ', '.join(double)))
    argstr_function = re.sub(r'([^\'])-([^\'])', r'\1_\2', ', '.join(arg_req + [x + "='-'" for x in arg_opt])) \
        .replace(', def=', ', drm=')
    argstr_process = ', '.join(arg_req + arg_opt) \
        .replace('-', '_') \
        .replace(', def,', ', drm,')
    doc_start = 'input parameters:[ ]*\n' if re.search('input parameters', out) else 'usage:.*(?=\n)'
    indent = ' ' * 4
    docstring_elements = ['Parameters\n----------']
    starts = []
    for x in arg_req + arg_opt:
        try:
            starts.append(re.search(r'\n[ ]*{0} .*'.format(x), out).start())
        except AttributeError:
            raise RuntimeError('cannot find parameter {}'.format(x))
    starts += [len(out)]
    pattern = r'\n[ ]*(?P<par>{0})[ ]+(?P<doc>.*)'.format('|'.join(arg_req + arg_opt))
    for i in range(0, len(starts) - 1):
        doc_raw = out[starts[i]:starts[i + 1]]
        match = re.match(pattern, doc_raw, flags=re.DOTALL)
        if not match:
            continue
        par = match.group('par')
        doc_items = re.split('\n+\s*', match.group('doc').strip('\n'))
        doc_items = [x.replace('*', '\*') for x in doc_items]
        latest = None
        for i in range(len(doc_items)):
            item = doc_items[i]
            if re.search('^(?:(?:-|)[-0-9]+|NOTE):', item):
                latest = i
                doc_items[i] = '* ' + re.sub(r'((?:-|)[-0-9]+:)(\w+)', r'\1 \2', item)
        if latest:
            if latest + 2 <= len(doc_items):
                i = 1
                while latest + i + 1 <= len(doc_items):
                    doc_items[latest + i] = '  ' + doc_items[latest + i]
                    i += 1
            else:
                doc_items[-1] = doc_items[-1] + '\n'
        description = '\n{0}{0}'.join(doc_items).format(indent)
        doc = '{0}:\n{1}{2}'.format(par, indent, description)
        docstring_elements.append(doc)
    doc = 'logpath: str or None\n{0}a directory to write command logfiles to'.format(indent)
    docstring_elements.append(doc)
    fun_def = 'def {name}({args_fun}, logpath=None):' \
        .format(name=os.path.basename(command).replace('-', '_'),
                args_fun=argstr_function)
    fun_doc = '\n{header}\n\n{doc}\n' \
        .format(header=header,
                doc='\n'.join(docstring_elements))
    fun_proc = "process(['{command}', {args_cmd}], logpath=logpath)" \
        .format(command=command,
                args_cmd=argstr_process)
    fun = '''{defn}\n"""{doc}"""\n{proc}'''.format(defn=fun_def, doc=fun_doc, proc=fun_proc)
    fun = fun.replace('\n', '\n{}'.format(indent)) + '\n'
    return fun
def parse_command ( command ) : command = which ( command ) proc = sp . Popen ( command , stdin = sp . PIPE , stdout = sp . PIPE , stderr = sp . PIPE , universal_newlines = True ) out , err = proc . communicate ( ) out += err header = '\n' . join ( [ x . strip ( '* ' ) for x in re . findall ( '[*]{3}.*[*]{3}' , out ) ] ) header = '| ' + header . replace ( '\n' , '\n| ' ) usage = re . search ( 'usage:.*(?=\n)' , out ) . group ( ) arg_req = [ re . sub ( '[^\w.-]*' , '' , x ) for x in re . findall ( '[^<]*<([^>]*)>' , usage ) ] arg_opt = [ re . sub ( '[^\w.-]*' , '' , x ) for x in re . findall ( '[^[]*\[([^]]*)\]' , usage ) ] for arg in arg_req + arg_opt : for item in re . findall ( arg , out , re . IGNORECASE ) : if item != arg : out = out . replace ( item , arg ) double = [ k for k , v in Counter ( arg_req + arg_opt ) . items ( ) if v > 1 ] if len ( double ) > 0 : raise RuntimeError ( 'double parameter{0}: {1}' . format ( 's' if len ( double ) > 1 else '' , ', ' . join ( double ) ) ) argstr_function = re . sub ( r'([^\'])-([^\'])' , r'\1_\2' , ', ' . join ( arg_req + [ x + "='-'" for x in arg_opt ] ) ) . replace ( ', def=' , ', drm=' ) argstr_process = ', ' . join ( arg_req + arg_opt ) . replace ( '-' , '_' ) . replace ( ', def,' , ', drm,' ) doc_start = 'input parameters:[ ]*\n' if re . search ( 'input parameters' , out ) else 'usage:.*(?=\n)' indent = ' ' * 4 docstring_elements = [ 'Parameters\n----------' ] starts = [ ] for x in arg_req + arg_opt : try : starts . append ( re . search ( r'\n[ ]*{0} .*' . format ( x ) , out ) . start ( ) ) except AttributeError : raise RuntimeError ( 'cannot find parameter {}' . format ( x ) ) starts += [ len ( out ) ] pattern = r'\n[ ]*(?P<par>{0})[ ]+(?P<doc>.*)' . format ( '|' . join ( arg_req + arg_opt ) ) for i in range ( 0 , len ( starts ) - 1 ) : doc_raw = out [ starts [ i ] : starts [ i + 1 ] ] match = re . match ( pattern , doc_raw , flags = re . DOTALL ) if not match : continue par = match . group ( 'par' ) doc_items = re . split ( '\n+\s*' , match . group ( 'doc' ) . strip ( '\n' ) ) doc_items = [ x . replace ( '*' , '\*' ) for x in doc_items ] latest = None for i in range ( len ( doc_items ) ) : item = doc_items [ i ] if re . search ( '^(?:(?:-|)[-0-9]+|NOTE):' , item ) : latest = i doc_items [ i ] = '* ' + re . sub ( r'((?:-|)[-0-9]+:)(\w+)' , r'\1 \2' , item ) if latest : if latest + 2 <= len ( doc_items ) : i = 1 while latest + i + 1 <= len ( doc_items ) : doc_items [ latest + i ] = '  ' + doc_items [ latest + i ] i += 1 else : doc_items [ - 1 ] = doc_items [ - 1 ] + '\n' description = '\n{0}{0}' . join ( doc_items ) . format ( indent ) doc = '{0}:\n{1}{2}' . format ( par , indent , description ) docstring_elements . append ( doc ) doc = 'logpath: str or None\n{0}a directory to write command logfiles to' . format ( indent ) docstring_elements . append ( doc ) fun_def = 'def {name}({args_fun}, logpath=None):' . format ( name = os . path . basename ( command ) . replace ( '-' , '_' ) , args_fun = argstr_function ) fun_doc = '\n{header}\n\n{doc}\n' . format ( header = header , doc = '\n' . join ( docstring_elements ) ) fun_proc = "process(['{command}', {args_cmd}], logpath=logpath)" . format ( command = command , args_cmd = argstr_process ) fun = '''{defn}\n"""{doc}"""\n{proc}''' . format ( defn = fun_def , doc = fun_doc , proc = fun_proc ) fun = fun . replace ( '\n' , '\n{}' . format ( indent ) ) + '\n' return fun
Parse the help text of a Gamma command to a Python function including a docstring.
Parse the help text of a Gamma command to a Python function including a docstring .
"""
    Parse the help text of a Gamma command to a Python function including a docstring.
    The docstring is in rst format and can thu be parsed by e.g. sphinx.
    This function is not intended to be used by itself, but rather within function :func:`parse_module`.

    Parameters
    ----------
    command: str
        the name of the gamma command

    Returns
    -------
    str
        the full Python function text

    """ # filter header command description and usage description text # filter required and optional arguments from usage description text # fix inconsistencies in parameter naming related to case differences, # e.g. ISP_PAR in the usage text vs. ISP_Par in the parameter description # print('header_raw: \n{}\n'.format(header)) # print('usage_raw: \n{}\n'.format(usage)) # print('required args: {}\n'.format(', '.join(arg_req))) # print('optional args: {}\n'.format(', '.join(arg_opt))) # print('double args: {}\n'.format(', '.join(double))) # create the function argument string for the Python function # optional arguments are parametrized with '-' as default value, e.g. arg_opt='-' # a '-' in the parameter name is replaced with '_' # example: "arg1, arg2, arg3='-'" # create the process call argument string # a '-' in the parameter name is replaced with '_' # e.g. 'arg1, arg2, arg3' # if a parameter is named 'def' (not allowed in Python) it is renamed to 'drm' # print('arg_str1: \n{}\n'.format(argstr_function)) # print('arg_str2: \n{}\n'.format(argstr_process)) # define the start of the parameter documentation string, which is either after 'input_parameters' or after # the usage description string # parse the parameter documentation to a Python docstring format # define the number of spaces to indent # gather the indices, which mark the documentation start of the respective parameters within  # the raw documentation text # define a pattern for parsing individual parameter documentations # print(pattern) # draw a subset from the Gamma docstring containing only the doc of a single parameter # print(repr(doc_raw)) # parse the docstring # retrieve the parameter name and the documentation lines # escape * characters (which are treated as special characters for bullet lists by sphinx) # convert all lines starting with an integer number or 'NOTE' to bullet list items # prepend '* ' and replace missing spaces after a colon: 'x:x' -> 'x: x' # format documentation lines coming after the last bullet list item # sphinx expects lines after the last bullet item to be indented by two spaces if # they belong to the bullet item or otherwise a blank line to mark the end of the bullet list # case if there are still lines coming after the last bullet item, # prepend an extra two spaces to these lines so that they are properly # aligned with the text of the bullet item # if not, then insert an extra blank line # parse the final documentation string for the current parameter # create docstring for parameter logpath # create the function definition string # create the complete docstring # create the process call string # combine the elements to a complete Python function string # indent all lines and add an extra empty line at the end
command