"""
Tools used during configuration and build process of the documentation
(mostly "bridges" between doxygen, breathe and sphinx)

This file is to be copied into CMAKE_BINARY_DIR/share using configure_file
"""

""" Siconos is a program dedicated to modeling, simulation and control
 of non smooth dynamical systems.

 Copyright 2018 INRIA.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""

import glob
import os
import subprocess
import shutil
import buildtools as bt
import pickle
import inspect
import importlib
import textwrap
import re
import operator

try: 
    import parse_doxygen as pxml
except:
    pass


# Needed only when WITH_DOXY2SWIG is ON
try:
    from sicodoxy2swig import SiconosDoxy2Swig
except:
    pass

try:
    import lxml.etree as ET
except ImportError:
    import xml.etree.ElementTree as ET

def parse_doxygen_config(filename):
    """Read doxygen config into a python dictionnary
    
    Parameters
    ----------
    filename : string
       full path to doxygen file
    
    Returns a python dictionnary
    """
    result = {}
    with open(filename) as ff:
        # remove comment lines
        conf = [n.strip() for n in ff.readlines() if (not n.startswith('#') and not n.startswith('\n'))]
        for d in conf:
            if d.count('=') > 0:
                res = d.split('=')
                result[res[0].rsplit()[0]] = res[1]
                backup = res.copy()
            else:
                result[backup[0].rsplit()[0]] += d
    return result
    
def get_xml_files(header_name, xml_path, case_sense_names=True):
    """Return all xml files generated by doxygen from a given header file.
    and found in a given path.

    Parameters
    ----------
    header_name : string
        header file.
    xml_path : string
        absolute path to xml files.
    case_sense_names : bool
        false if xml output files names are lower case only

    Returns a list of xml files.
    
    Notes
    -----
 
    * This function takes into account the value of CASE_SENSE_NAMES parameter
    in doxygen config.
    * It looks for '*classNAME.xml', '*structNAME.xml' and 'NAME_8h*.xml'
    name being the input header name (without ext).

    """
    # Get filename without extension
    fnwe = os.path.basename(header_name).split('.')[0]
    # Replace _ with __ in filename
    fnwe = fnwe.replace('_', '__')
    if not case_sense_names:
        fnwe = replace_uppercase_letters(fnwe)
    # Look for 'class' files
    classfiles = glob.glob(os.path.join(xml_path, 'class' + fnwe + '.xml'))
    # Look for 'struct' files
    structfiles = glob.glob(os.path.join(xml_path, 'struct' + fnwe + '.xml'))
    # Look for '8h' (?) files
    files8h = glob.glob(os.path.join(xml_path, fnwe + '_8h*.xml'))
    allfiles = classfiles + structfiles + files8h
    return allfiles



def xml2swig(header_name, component_name, xml_path, swig_working_dir,
             case_sense_names, docstrings_features, all_index):
    """For a given header file, create swig (.i) file using 
    xml outputs from doxygen (Driver : Doxy2SWIG)

    Parameters
    ----------
    header_name : string
        Name of the header file (h or hpp)
    component_name : string
         component (numerics, kernel, ...) of interest
    xml_path : string
        absolute path to xml files.
    swig_working_dir : string
        Absolute path to siconos swig outputs
    case_sense_names : bool
        false if xml output files names are lower case only
    docstrings_features: dict
        all docstrings features (keys) with the header used to build them (values)
    all_index: dict
        class and files names (keys) with a short description (values) (in-out)

     Notes
     -----
     * This function takes into account the value of CASE_SENSE_NAMES parameter
     in doxygen config.
     * .i file will be named as the xml input file, with .i as ext.
     * all latex formula are replaced with temporary strings and postprocess later)

    """
    # Get xml files related to current header
    allfiles = get_xml_files(header_name, xml_path, case_sense_names)

    # Build .i files
    for f in allfiles:
        filter_dot_in_xml_formulas(f)
        # set output filename == xml file without extension + .i
        p = SiconosDoxy2Swig(f, component_name, swig_working_dir)
        p.run()
        featname, kind, descr = p.get_xml_compound_infos()
        if kind == 'class' or kind == 'struct':
            all_index[featname] = descr
        elif kind == 'file':
            all_index[header_name.split('/')[-1]] = descr
            
        for feat in p.features:
            docstrings_features[feat] = header_name
        if len(p.enums) > 0:
            docstrings_features['pydata_' + p.name] = p.enums
            
    # Look for namespaces files
    namespaces_files = glob.glob(os.path.join(xml_path, 'namespace*.xml'))
    # Build corresponding .i files
    for f in namespaces_files:
        namespace_name = f.split('.')[0].split('namespace')[-1]
        namespace_name = namespace_name.replace(r'_1_1', r'::')
        filter_dot_in_xml_formulas(f)
        # set output filename == xml file without extension + .i
        p = SiconosDoxy2Swig(f, component_name, swig_working_dir)
        p.run()
        hpp_name = p.get_specific_subnodes(p.xmldoc, 'location', recursive=4)
        hpp_name = hpp_name[0].attributes['file'].value
        featname, kind, descr = p.get_xml_compound_infos()
        if kind == 'class' or kind == 'struct':
            all_index[featname] = descr
        elif kind == 'file':
            all_index[hpp_name.split('/')[-1]] = descr
        for feat in p.features:
            docstrings_features[feat] = hpp_name
        if len(p.enums) > 0:
            docstrings_features['pydata_' + p.name] = p.enums

            
def replace_uppercase_letters(filename):
    """Replace uppercase letters in a string
    with _lowercase (following doxygen way)
    
    e.g. : TimeStepping --> _time_stepping

    This is useful to postprocess filenames from xml-doxygen outputs
    and feed them to doxy2swig, even when CASE_SENSE_NAMES = NO
    in doxygen config.

    Usage:

    result = replace_uppercase_letters(input)
      
    """
    r = []
    for c in filename:
        # l being: last character was not uppercase
        newc = c
        if c.isupper():
            newc ='_' + c.lower()
        r.append(newc)
    return ''.join(r)


def filter_doxygen_warnings_files(warnings_path, outputfile):
    """Post process outputs from doxygen warnings : 
     * remove all empty files
     * cat all warnings files into one

     Parameters
     ----------
     warnings_path : string
         full path to warnings files
     outputfile : string
         name of resulting file (concat.). Saved in warnings_path
    """
    
    warnfiles = glob.glob(os.path.join(warnings_path, '*.warnings'))
    real_warnings = []
    for f in warnfiles:
        if os.stat(f).st_size == 0:
            os.remove(f)
        else:
            real_warnings.append(f)
    outputfile = os.path.join(warnings_path, outputfile)
    with open(outputfile, 'w') as outfile:
        for fname in real_warnings:
            with open(fname) as infile:
                for line in infile:
                    outfile.write(line)
            
    
def parse_doxygen_wrapper(xml_directory, rst_directory):

    # OUTDATED - TEST PURPOSE (parse_doxygen test : https://bitbucket.org/trlandet/doxygen_to_sphinx_and_swig)
    
    # Look for 'namespaces
    namespacefiles = glob.glob(os.path.join(xml_directory, 'namespace*.xml'))
    xmlnamespaces = []
    for f in namespacefiles:
        name = os.path.basename(f).split('.')[0]
        name = name.split('namespace')[1]
        xmlnamespaces.append(name)

    all_rst_files = []
    all_swig_files = []
    for namespace in xmlnamespaces:
        namespaces = pdox.read_doxygen_xml_files(xml_directory, [namespace])
        # Get sorted list of members
        members = list(namespaces[namespace].members.values())
        members.sort(key=lambda m: m.name)
    
        # Make Sphinx documentation
        filename = namespace + '_api.rst'
        with open(filename, 'wt') as out:
            for member in members:
                out.write(member.to_rst())
                out.write('\n')
            out.write('\n')
            all_rst_files.append(filename)
            
        # Make SWIG interface file
        filename = namespace + '_docstrings.i' 
        with open(filename, 'wt') as out:
            out.write('// SWIG docstrings generated by doxygen and parse_doxygen.py\n\n')
            for member in members:
                out.write(member.to_swig())
                out.write('\n')
            out.write('\n')
            all_swig_files.append(filename)
    
        for tag in pdox.NOT_IMPLEMENTED_ELEMENTS:
            print('WARNING: doxygen XML tag %s is not supported by the parser' % tag)
            
        docstringsfile = 'generared_docstrings.i'
        sphinxfile = os.path.join(rst_directory, 'generated_api.rst')
        with open(sphinxfile, 'w') as outfile:
            header ='doxy to rst\n-----------\n\n'
            outfile.write(header)

            for fname in all_rst_files:
                namesp = fname.split('_api.rst')[0]
                line= namesp + '\n-----------\n\n'
                outfile.write(line)
                
                with open(fname) as infile:
                    for line in infile:
                        outfile.write(line)
        with open(docstringsfile, 'w') as outfile:
            for fname in all_swig_files:
                with open(fname) as infile:
                    for line in infile:
                        outfile.write(line)
  


def build_docstrings(headers, component_name, doxygen_config_filename, siconos_swig_path):
    """Create docstrings (doxy2swig) in swig files from xml (doxygen) generated
    from headers.

    Parameters
    ----------

    headers : list (cmake like)
         headers files to parse
    component_name : string
         component (numerics, kernel, ...) of interest
    doxygen_config_filename : string
         name (full path) of the doxygen configuration file
    siconos_swig_path : string
         path to swig outputs in binary dir (i.e. wrap)

    Note
    ----
    * all swig files will be genereted into siconos_wig_path/tmp_component_name directory
    and concatenated into component_name-docstrings.i that will be the file really
    used by swig.
    """
    doxyconf = parse_doxygen_config(doxygen_config_filename)
    case_sense_names = doxyconf['CASE_SENSE_NAMES'].find('YES') > -1 
    xml_path = os.path.join(doxyconf['OUTPUT_DIRECTORY'].lstrip(), doxyconf['XML_OUTPUT'].lstrip())
    headers = bt.parse_cmake_list(headers)
    swig_working_dir = os.path.join(siconos_swig_path, 'tmp_' + component_name)
    if not os.path.exists(swig_working_dir):
        os.makedirs(swig_working_dir)

    docstrings_features = {}
    all_index = {}
    for hfile in headers:
        xml2swig(hfile, component_name, xml_path, swig_working_dir,
                 case_sense_names, docstrings_features, all_index)

    # Save features info in pickle file
    pickle_filename = os.path.join(siconos_swig_path, component_name + '.pickle')
    with open(pickle_filename, 'wb') as currentfile:
        pickle.dump(docstrings_features, currentfile)
    # Save class and files names and descriptions (for toc)
    pickle_filename = os.path.join(siconos_swig_path, component_name + '_index.pickle')
    with open(pickle_filename, 'wb') as currentfile:
        pickle.dump(all_index, currentfile)

    outputfile = os.path.join(siconos_swig_path, component_name + '-docstrings.i')
    swigfiles = glob.glob(os.path.join(swig_working_dir, '*.i'))
    with open(outputfile, 'w', encoding='utf8') as outfile:
        for fname in swigfiles:
            with open(fname, encoding='utf8') as infile:
                for line in infile:
                    outfile.write(line)
    msg = 'Generates file ' # + outputfile + ' for doctrings in swig.'
    print(msg)


def create_breathe_files(headers, srcdir, component_name, sphinx_directory, doxygen_config_filename):
    """Create rst files for sphinx from xml (doxygen) outputs generated from headers.

    Parameters
    ----------

    headers : list (cmake like)
         headers files to parse
    srcdir : string
        absolute path to c/c++ sources (CMAKE_SOURCE_DIR)
    component_name : string
         component (numerics, kernel, ...) of interest
    sphinx_directory : string
        directory where rst files will be written
    doxygen_config_filename : string
         name (full path) of the doxygen configuration file

    Notes:
    * for each header, rst files (class, struct, file and source codes) will be generated
    * three other 'main' rst files will be produced :
       * breathe_api.rst, with the toctree for all classes and structs
       * files_list.rst, with the toctree for all files documentation
       * sources_list.rst with the toctree for all program listings

    """

    # Get all headers for the current component
    headers = bt.parse_cmake_list(headers)
    rst_files = []
    rst_programs = []
    # Parse doxygen config
    doxyconf = parse_doxygen_config(doxygen_config_filename)
    xmlconf = {}
    sphinx_directory = os.path.join(sphinx_directory, component_name)
    if not os.path.exists(sphinx_directory):
        os.makedirs(sphinx_directory)
    
    xmlconf['CASE_SENSE_NAMES'] = doxyconf['CASE_SENSE_NAMES'].find('YES') > -1 
    xmlconf['XML_OUTPUT'] = os.path.join(doxyconf['OUTPUT_DIRECTORY'].lstrip(), doxyconf['XML_OUTPUT'].lstrip())

    all_index = {}
    # -- Create rst files for classes, structs and files found in xml directory --
    for hfile in headers:
        xml2rst(hfile, srcdir, component_name, sphinx_directory, xmlconf, all_index)

    # -- Create rst files to collect list of classes and files (i.e. files just created above) --
    class_and_struct_files = glob.glob(os.path.join(sphinx_directory, 'class*.rst'))
    class_and_struct_files += glob.glob(os.path.join(sphinx_directory, 'struct*.rst'))
    class_and_struct_files.sort()
    pgm_files = glob.glob(os.path.join(sphinx_directory, 'pgm_*.rst'))
    pgm_files.sort()
    rst_files = glob.glob(os.path.join(sphinx_directory, 'file_*.rst'))
    rst_files.sort()
    all_files = class_and_struct_files + rst_files
    all_files.sort()
    # -- Name of the main rst files for the current component --
    # usually : docs/sphinx/reference/cpp/component_name/autodoc_all.rst
    outputname = os.path.join(sphinx_directory, 'autodoc_all.rst')
    #title = component_name + ' component\n'
    #title += len(title) * '=' + '\n\n'
    indent = 4 * ' '
    basename = '/reference/cpp/' + component_name + '/'
    with open(outputname, 'wt') as out:
        opt = ':maxdepth: 4\n'
        for f in all_files:
            name = os.path.basename(f).split('.')[0]
            if name.find('class') > -1 :
                shorttitle = name.split('class')[-1]
                text = '* :class:`' + shorttitle + '` : '
            elif name.find('struct') > -1:
                shorttitle = name.split('struct')[-1]
                text = '* :class:`' + shorttitle + '` : '
            elif name.find('file_') > -1:
                shorttitle = name.split('file_')[-1].replace('_', '.')
                name = basename + name
                text = '* :doc:`' + shorttitle + '<' + name + '>` : '               
            try:
                text += all_index[shorttitle] + '\n'
            except:
                text += '\n'
            name = basename + name
            #gen = textwrap.indent(text, 4 * ' ')
            out.write(text)
        out.write('\n\n')    

    # Classes and structs
    outputname = os.path.join(sphinx_directory, 'autodoc_classes.rst')
    subtitle = 'Classes and structs'
    subtitle += '\n' + len(subtitle) * '-' + '\n\n'
    with open(outputname, 'wt') as out:
        #out.write(title)
        out.write(subtitle)
        #out.write('.. toctree::\n')
        opt = ':maxdepth: 4\n'
        #out.write(textwrap.indent(opt, indent))
        #out.write('\n')  
        for f in class_and_struct_files:
            name = os.path.basename(f).split('.')[0]
            if name.find('class') > -1 :
                shorttitle = name.split('class')[-1]
            elif name.find('struct') > -1:
                shorttitle = name.split('struct')[-1]
            text = '* :class:`' + shorttitle + '` : '
            try:
                text += all_index[shorttitle] + '\n'
            except:
                text += '\n'
            name = basename + name
            #gen = textwrap.indent(text, 4 * ' ')
            out.write(text)
        out.write('\n\n')    
    # Files doc
    outputname = os.path.join(sphinx_directory, 'autodoc_files.rst')
    subtitle = 'Files documentation\n'
    subtitle += len(subtitle) * '-' + '\n\n'
    with open(outputname, 'wt') as out:
        out.write(subtitle)
        #out.write('.. toctree::\n    :maxdepth: 2\n\n')
        for f in rst_files:
            name = os.path.basename(f).split('.')[0]
            shorttitle = name.split('file_')[-1].replace('_', '.')
            name = basename + name
            text = '* :doc:`' + shorttitle + '<' + name + '>` : '
            try:
                text += all_index[shorttitle] + '\n'
            except:
                text += '\n'
            #gen = textwrap.indent(shorttitle + '<' + name + '>\n', 4 * ' ')     
            out.write(text)
        out.write('\n\n')     
    # Programs listings
    outputname = os.path.join(sphinx_directory, 'autodoc_pgm.rst')
    label = '.. _' + component_name + '_pgm_listings:\n\n'
    title = component_name.title() + ' programs listings\n'
    title += len(title) * '-' + '\n\n'
    title = label + title
    with open(outputname, 'wt') as out:
        out.write(title)
        out.write('.. toctree::\n    :maxdepth: 2\n\n')
        for f in pgm_files:
            name = os.path.basename(f).split('.')[0]
            shorttitle = f.split(sphinx_directory)[-1].split('.')[0]
            splits = shorttitle.split('_')
            shorttitle = '/'.join(splits[2:-1]) + '.' + splits[-1]
            name = basename + name
            gen = textwrap.indent(shorttitle + '<' + name + '>\n', 4 * ' ')     
            out.write(gen)


def xml2rst(headername, srcdir, component_name, sphinx_directory, doxyconf, all_index):
    """Generate rst file(s) from xml (doxygen outputs) for given C/C++ header

    Parameters
    ----------
    headername : string
        name of the header (full path)
    srcdir : string
        absolute path to c/c++ sources (CMAKE_SOURCE_DIR)
    component_name : string
         component (numerics, kernel, ...) of interest
         (i.e. breathe project)
    sphinx_directory : string
        directory where rst files will be written
    doxyconf : dict
        dict describing xml/doxy conf.    
    """
    case_sense_names = doxyconf['CASE_SENSE_NAMES']
    xml_path = doxyconf['XML_OUTPUT']
    # First get list of xml files generated from current header by doxygen
    xml_files = get_xml_files(headername, xml_path, case_sense_names)
    # Then, for each xml, write sphinx header.
    # 3 cases : class, struct or file.
    for f in xml_files:
        
        filter_dot_in_xml_formulas(f)
        path = os.path.join(xml_path, f)
        root = ET.parse(path).getroot()
        compounds = root.findall('compounddef')
        refname = sphinxref4headername(headername, srcdir)
        outputname = os.path.basename(headername).split('.')[0]
        name, kind, descr = get_xml_compound_infos(compounds[0])
        if f.find('class') > -1 or f.find('struct') > -1:
            assert len(compounds) == 1
            all_index[name] = descr
            assert kind == 'struct' or kind == 'class'
            label = '.. _' + kind +'_' + name + ':\n\n'
            title = kind.title() + ' ' + name
            lenname = len(title)
            title = label + title + '\n' + lenname * '-' + '\n\n'
            pgm = 'Defined in :ref:`pgm' + refname + '`' + '\n\n'
            gen = title + pgm            
            gen +='.. doxygen' + kind + ':: ' + name + '\n'
            gen += '     :project: ' + component_name + '\n'
            outputname = kind + os.path.basename(headername).split('.')[0]
            outputname = os.path.join(sphinx_directory, outputname + '.rst')

        elif f.find('_8h') > -1:
            label = '.. _file' + refname + ':\n\n'
            shortname = headername.split(srcdir)[-1]
            shortname = shortname.replace('/./', '/')
            if shortname[0] == '/':
                shortname = shortname[1:]
            title = 'File ' + shortname
            lenname = len(title)
            title = label + title + '\n' + lenname * '-' + '\n\n'
            pgm = ':ref:`Go to the source code of this file <pgm' + refname + '>`' + '\n\n'
            gen = title + pgm
            sphinx_root = os.path.join(sphinx_directory, '../../')
            #relpath = os.path.relpath(srcdir, sphinx_root)
            #fname = os.path.join(relpath, shortname[1:])
            gen += '.. doxygenfile:: ' + shortname + '\n'
            gen += '     :project: ' + component_name + '\n'
            outputname = 'file_' + os.path.basename(headername).replace('.', '_')
            outputname = os.path.join(sphinx_directory, outputname + '.rst')       
            fname = shortname.split('/')[-1]
            all_index[fname] = descr
            
        else: # namespaces files.
            # Nothing to be done, breathe deal with those
            # directly from _8h file.
            continue
            
        with open(outputname, 'wt') as out:
            out.write(gen)
            out.write('\n')
            
    create_rst_for_program(headername, srcdir, sphinx_directory, True)


def module_docstrings2rst(component_name, module_path, module_name,
                          sphinx_directory, swig_working_dir):
    """Import a module and create 'rst' (autodoc)
    file for each documented (docstrings) object.

    Parameters
    ----------

    component_name : string
         name of the current component (e.g. kernel)
    module_path : string
        current module path, relative to swig working dir
        (usually wrap/siconos), e.g. mechanics/collision for module bodies.
    module_name : string
         name of the module (e.g. sensor)
    sphinx_directory : string
         directory (absolute) where rst files will
         be written. 
    swig_working_dir : string
         directory (absolute) where python modules
         are generated using swig

    Notes:
       * module_path is required for module like sensor
         located in siconos/control directory, 
         to build module name like siconos.control.sensor
       * Usually : sphinx_directory 
         = binary_dir/docs/sphinx/reference/python/module_name
       * Results : 
          * pyclass*.rst or pyfunction*.rst in sphinx_directory
          * pyclass.rst to collect (docstree toc) all pyclass*
          * pyfunctions.rst to collect all pyfunc *
  
    """

    # Test case with submodules (e.g. sensor in control)
    if module_path == '.' or module_path == '':
         module_name = 'siconos.' + module_name
    else:
        module_path = module_path.replace(r'/', r'.')
        module_name = 'siconos.' + module_path + '.' + module_name

    comp = importlib.import_module(module_name)
    sphinx_directory = sphinx_directory + module_name.replace(r'.', '_') 
    if not os.path.exists(sphinx_directory):
        os.makedirs(sphinx_directory)
        
    features_filename = comp.__file__.split(component_name)[0] + component_name + '.pickle'
    with open(features_filename, 'rb') as f:
        features = pickle.load(f)

    # We have to remove pyfiles each time the function is call
    # because of 'a' (append) in writing process
    pyfiles = glob.glob(os.path.join(sphinx_directory, '*pyfile.rst'))
    pyfiles += glob.glob(os.path.join(sphinx_directory, '*pyclass.rst'))
    for file in pyfiles:
        os.remove(file)

    class_files = []
    pyfunc_files = []
    objlist = [obj for obj in dir(comp) if not obj.startswith('_')]
    for obj in objlist:
        current = getattr(comp, obj)
        needs_doc = hasattr(current, '__doc__') and current.__doc__ is not None 
        if needs_doc and len(current.__doc__.strip()) > 0:
            gen, kind, name = create_autodoc(current, module_name)
            if kind == 'pyclass': # one rst file per class
                outputname = name + '_' + kind + '.rst'
                outputname = os.path.join(sphinx_directory, outputname)
                class_files.append(outputname)
            elif kind == 'pyfunction': # one rst file for all functions from a given header
                if name in features:
                    featname = features[name]
                elif name.replace('_', '::', 1) in features:
                    # - when two classes wrapped with swig have
                    # the same method (same name),
                    # swig create class1_methodname and class2_methodname.
                    # while features name is class1::methodname.
                    # - the same kind of thing happens for static class methods.
                    # We have to take these into account ... and the fact
                    # that some methods names may contain '_'
                    # (maxreplace=1 in replace below)
                    featname = features[name.replace('_', '::', 1)]
                elif name.split('::')[-1] in features:
                    # another way for swig to deal with
                    # namespaces ...
                    featname = features[name.split('::')[-1]]
                    
                else:
                    keys = list(features.keys())
                    for k in keys:
                        if k.count(name) > -1:
                            featname = features[k]
                            break
                    else:
                        raise Exception('Unknown feature name : ', name)
                    #featname = name
                outputname = os.path.basename(featname).split('.')[0]
                outputname = outputname + '_pyfile.rst'
                outputname = os.path.join(sphinx_directory, outputname)
                pyfunc_files.append(outputname)
            else:
                # pydata, processed later.
                continue
            with open(outputname, 'a+') as out:
                out.write(gen)
                out.write('\n')

    # -- Create rst files to collect list of classes and files (i.e. files just created above) --
    #class_files.sort()
    pyfunc_files = list(set(pyfunc_files))
    #pyfunc_files.sort()

    # Insert title (required to be taken into account in toctree ...)
    for fname in pyfunc_files:
        shortname = os.path.basename(fname).split('.')[0]
        label = '.. _' + shortname + ':\n\n'

        title = shortname.split('_pyfile')[0] + ' (functions)' 
        lenname = len(title)
        title = label + title + '\n' + lenname * '-' + '\n\n'
        with open(fname, 'r+') as f:
            lines = f.read()
            f.seek(0,0)
            f.write(title + lines)
    
    all_index_filename = os.path.join(swig_working_dir, component_name + '_index.pickle')
    with open(all_index_filename, 'rb') as f:
        all_index = pickle.load(f)
        
    allfiles = class_files + pyfunc_files
    allfiles.sort()
    submodule_name = module_name.split('.')[-1]
    outputname = os.path.join(sphinx_directory, 'autodoc_all.rst')
    title = module_name + '\n'
    title += len(title) * '=' + '\n\n'
    indent = 4 * ' '
    basename = '/reference/python/' + module_name.replace(r'.', '_')

    # A few lines to illustrate module usage
    header = '**Usage example** :\n\n.. code-block:: python\n\n'
    importname = 's' + module_name.split('.')[-1][0]
    code = 'import ' + module_name + ' as ' + importname
    code += '\n \nhelp(' + importname  + '.SomeClass)\n\n'
    header += textwrap.indent(code, '    ')
    header += '**Classes and functions**\n\n'
    
    with open(outputname, 'wt') as out:
        out.write(title)
        out.write(header)
        #out.write('.. toctree::\n    :maxdepth: 2\n\n')
        gen = '* :doc:`Enums and constants <' + os.path.join(basename, 'autodoc_pydata')+ '>`\n'
        for f in allfiles:
            name = os.path.basename(f).split('.')[0]
            text = ''
            if name.count('_pyclass') > 0:
                realname = name.split('_pyclass')[0] 
                shorttitle = realname + ' (class) '
                #text = '* :doc:`' + shorttitle + '<' + basename + name + '>` : '
                text = '* :py:class:`' + module_name + '.' + realname + '` : '
                try:
                    text += all_index[realname] + '\n'
                except:
                    text += ' \n'
            elif name.count('_pyfile') > 0:
                realname = name.split('_pyfile')[0] 
                shorttitle = realname + ' (functions) '
                text = '* :doc:`' + shorttitle + '<' + os.path.join(basename, name) + '>` : '
                if realname + '.h' in all_index:
                    text += all_index[realname + '.h'] + ' \n'
                elif realname + '.hpp' in all_index:
                    text += all_index[realname + '.hpp'] + ' \n'
                else:
                    text += ' \n'
            else:
                shorttitle = ''

            gen += text
        # out.write(textwrap.indent(gen, indent))
        out.write(gen + '\n')
    
    # It might be necessary to parse some latex from doxygen and convert it to sphinx ...
    latex_dir = swig_working_dir + 'tmp_' + component_name
    replace_latex(outputname, latex_dir)

            
    # Process enums
    # Get saved enums for the current module
    outputname = os.path.join(sphinx_directory, 'autodoc_pydata.rst')
    title = module_name + ' constants (Python API)\n'
    title += len(title) * '-' + '\n\n'
    title += 'All the predefined global constants in ' + module_name
    title += '(generated from C++ enum, global variables, ...) \n\n'
    enumskeys = [k for k in features if k.find('pydata') > -1]
    enums = [features[key] for  key in enumskeys]
    header = '**Usage** :\n\n.. code-block:: python\n\n'
    importname = 's' + module_name.split('.')[-1][0]
    code = 'import ' + module_name + ' as ' + importname
    code += '\n \nprint(' + importname + '.CONSTNAME)\n\n'
    header += textwrap.indent(code, '    ')
    title += header
    title += '\n-----\n\n**List and descriptions of available constants** :\n\n'

    with open(outputname, 'wt') as out:
        out.write(title) 
        for key in enumskeys:
            enums = features[key]
            for ename in enums:
                # Document only data available in python API
                if hasattr(comp, ename):
                    # and only data with a description
                    if len(enums[ename][1].strip()) > 0:
                        gen = ''
                        gen += '.. _pydata_' + ename + ':\n\n'
                        gen += '.. py:data:: ' + ename + '\n\n'
                        if len(enums[ename][0]) > 0:
                            # Add initializer value if set
                            gen += '    {0} ({1})\n\n'.format(enums[ename][1].strip(), enums[ename][0])
                        else:
                            gen += '    {0} \n\n'.format(enums[ename][1].strip())
                        out.write(gen)            
    # It might be necessary to parse some latex from doxygen and convert it to sphinx ...
    replace_latex(outputname, latex_dir)

def create_autodoc(current, module_name):
    """Create autodoc directive
    
    Parameters
    ----------
    current : object
        'object' of interest, one output from python
        command dir(module)
    module_name : string
        module name (e.g. siconos.kernel)
    sphinx_directory : string
        location (full path) where output files
        should be written
    """
    if inspect.isclass(current):
        name = current.__name__
        fullname = module_name + '.' + name
        kind = 'pyclass'
        title = fullname + ' (Python class)'
        directive = '.. autoclass:: ' + fullname + '\n'
        directive += '    :members:\n\n'
        #label = '.. index:: single: ' + module_name.split('.')[1] + ';' + name
        #label += '\n'
        label = '.. _' + kind +'_' + name + ':\n\n'
        lenname = len(title)
        title = label + title + '\n' + lenname * '-' + '\n\n'
    elif inspect.isfunction(current):
        name = current.__name__
        fullname = module_name + '.' + name
        title = fullname
        kind = 'pyfunction'
        directive = '.. autofunction:: ' + fullname + '\n\n'
        #label = '.. index:: single: ' + module_name.split('.')[1] + ';' + name
        #label += '\n'
        label = '.. _' + kind +'_' + name + ':\n\n'
        title = label + 4 * '-' + '\n\n'
    else: # current is neither a class nor a function
        # try data ...
        kind = 'pydata'
        gen = ''#'.. autodata:: ' + str(current) + '\n\n'
        name = 'Unknown'
        return gen, kind, name

    gen = title + directive
    return gen, kind, name



siconos_components ={'externals' : 0, 'numerics': 1, 'kernel' : 2, 'control': 3, 'mechanics': 4, 'io':5}

def build_python_api_main(outputdir, common_header):

    mainrst_filename = os.path.join(outputdir, 'python_api.rst')
    # list documented (python) packages
    docpython_dir = os.path.join(outputdir, 'python')
    packages = glob.glob(os.path.join(docpython_dir, '*'))
    packages = [os.path.basename(p) for p in packages]
    # (ugly) trick to print components in the expected order.
    pack ={}
    for p in siconos_components:
        for pname in packages:
            if pname.count(p) > 0:
                pack[pname] = siconos_components[p]
    packages = [p[0] for p in sorted(pack.items(), key=operator.itemgetter(1))]
    with open(mainrst_filename, 'w') as f:
        label = '.. _siconos_python_reference:\n\n\n'
        title = 'Siconos Python API reference'
        title += '\n' + len(title) * '#' + '\n\n'
        title += 'This is the documentation of `python <https://www.python.org/>`_ interface to Siconos.\n\n\n'
        f.write(label)
        f.write(title)
        indent = 4 * ' '
        f.write(common_header)
        for p in packages:
            directive = '.. include:: python/' + p + '/autodoc_all.rst\n\n'
            f.write(directive)


def build_cpp_api_main(outputdir, common_header):

    mainrst_filename = os.path.join(outputdir, 'cpp_api.rst')
    # list documented (cpp) packages
    doccpp_dir = os.path.join(outputdir, 'cpp')
    packages = glob.glob(os.path.join(doccpp_dir, '*'))
    packages = [os.path.basename(p) for p in packages if os.listdir(p)]
    # trick to print components in the expected order.
    #components = [os.path.join(doccpp_dir, p) for p in siconos_components]
    packages = [p for p in siconos_components if p in packages]
    indent = 4 * ' '
    class_diag = 'Class diagrams (UML view)'
    class_diag += '\n' + len(class_diag) * '=' + '\n\n'
    class_diag += ':doc:`/reference/class_diagrams`\n\n'
    
    with open(mainrst_filename, 'w') as f:
        label = '.. _siconos_cpp_reference:\n\n\n'
        title = 'Siconos C/C++ API reference'
        title += '\n' + len(title) * '#' + '\n\n'
        title += 'This is the documentation of C/C++ interface to Siconos.\n\n\n'
        f.write(label)
        f.write(title)
        f.write(common_header)
        tab_directive = '.. csv-table::\n'
        tab_directive += textwrap.indent(':widths: 60 40\n\n', indent)
        column_titles = '**Classes and structs**, **Files**\n'
        tab_directive += textwrap.indent(column_titles, indent)

        f.write(class_diag)
        for p in packages:
            title = p.title() + ' component\n'
            title += len(title) * '=' + '\n\n'
            ppath = 'cpp/' + p
            f.write(title)
            pgm_listings = 'Check :ref:`' + p + '_pgm_listings`'
            pgm_listings += ' for a complete list of headers for this component.'
            f.write(pgm_listings + '\n\n')
            #f.write(tab_directive)
            directive = '.. include:: ' + ppath + '/autodoc_classes.rst'
            directive += '\n'#','
            directive += '.. include:: ' + ppath + '/autodoc_files.rst\n'
            indent = ''
            f.write(textwrap.indent(directive, indent))
            f.write('\n')


def build_api_main(sphinx_directory):
    """Write api files for python and C++

    Parse existing rst files (one for each class,
    + those for functions)
    and collect them into cpp_api.rst and python_api.rst
    in sphinx/reference directory.

    This function is called by cmake.
    """

    # Place where files will be written
    outputdir = os.path.join(sphinx_directory, 'reference')

    # Top file comments
    remark = '\n*If a file or a class you know does not appear in this page, '
    remark += 'it means it has not been (properly) documented or is not '
    remark += 'available in the high level API. Please contact us '
    remark += 'if you think it is an error.*\n\n'
    header = '.. contents::\n    :local:\n\n'
    introduction = 'Below you will find links to documentation for all classes and files in Siconos, '
    introduction += 'sorted  by component.\n\n'
    common_header = remark + introduction + header
    # Create main rst file for C/C++ API
    build_cpp_api_main(outputdir, common_header)
    # Create main rst file for Python API
    build_python_api_main(outputdir, common_header)
    
            
def create_rst_for_program(headername, srcdir, sphinx_directory, filterdox=False):
    """
    Parameters
    ----------
    headername : string
        name of the header (full path)
    srcdir : string
        absolute path to c/c++ sources (CMAKE_SOURCE_DIR)
    sphinx_directory : string
        directory where rst files will be written
    filterdox : boolean
        true to remove doxygen comments from program listings in sphinx
    """
    shortname = headername.split(srcdir)[-1][1:]
    shortname = shortname.replace(r'./', '')
    #outputname = 'pgm_' + os.path.basename(headername).split('.')[0]
    outputname = 'pgm_' + shortname.replace('/', '_').replace('.', '_')
    outputname = os.path.join(sphinx_directory, outputname + '.rst')
    refname = sphinxref4headername(headername, srcdir)
    
    title = 'Program listing for file ' + shortname
    label = '.. _pgm' + refname + ':\n\n'
    lenname = len(title)
    title = label + title + '\n' + lenname * '=' + '\n\n'
    doc = '* Return to documentation for :ref:`this file<file' + refname + '>`\n\n'

    gen = title + doc
    if filterdox:
        d = filter_comments(headername)
    else:
        with open(headername, 'r') as f:
            d = f.read()
        
    gen += '.. code-block:: c++\n'
    gen += '    :linenos:\n\n'
    text = textwrap.indent(d, 4 * ' ')
    gen += text

    with open(outputname, 'wt') as out:
        out.write(gen)

def get_xml_compound_infos(compound):
    """Returns object (class, struct ...) name, kind
    and brief description
    from an xml node (compound)
    """
    kind = compound.attrib['kind']
    names = compound.findall('compoundname')
    descr = compound.findall('briefdescription')
    assert len(names) == 1
    assert len(descr) == 1
    #res = ''
    content = descr[0].find('para')
    if content is not None:
        res = ET.tostring(content, method='text')
        # for child in descr[0]:
        #     try:
        #         res += child.text
        #     except:
        #         pass
        #     for sub in child:
        #         try :
        #             res += sub.text
        #         except:
        #             pass
        # new lines in description must be indented
        res = '\n    '.join(res.decode().split('\n'))
        res = find_and_replace_math(res)
    else:
        res = ''
    return names[0].text, kind, res

def find_and_replace_math(content):
    if content.count('$') > 1:
        start = content.find('$')
        end = content.find('$', start + 1)
        doxy_latex = content[start: end + 1]
        rst_latex = r':math:`' + doxy_latex[1:-1].strip() + r'`'
        content = content.replace(doxy_latex, rst_latex)
    return content



def sphinxref4headername(headername, srcdir):
    """Returns the reference to set in sphinx for a given file name
    
    Parameters
    ----------
    headername : string
        name of the input file (full path)
    srcdir : part of the name to remove from ref (usually CMAKE_SOURCE_DIR)

    
    """
    if len(srcdir) > 0:
        refname = headername.split(srcdir)[-1]
    else:
        refname = headername
    refname = refname.replace('/', '_')
    return refname

    
def filter_comments(headername):
    """Returns list of lines from a
    source file, without doxygen comments.
    """
    def replacer(match):
        s = match.group(0)
        if s.startswith('/'):
            return " " # note: a space and not an empty string
        else:
            return s

    with open(headername, 'r') as f:
        text = f.read()
        pattern = re.compile(
            r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
            re.DOTALL | re.MULTILINE
        )
        return re.sub(pattern, replacer, text)



def find_doxygen_diagrams(doxygen_path, output_directory):
    """Python utility to create sphinx rst file from png generated
    by doxygen (class diagrams ...)

    - scan doxygen ouput (html) path
    - create an image entry for each class_inh*.png file in a rst file

    Parameters
    ----------
    doxygen_path : string
       directory (full path) which contains png files generated by doxygen
    output_path : string
       directory (fullpath) of the rst output
    """
    
    # Scan doxygen output path and create a list with
    # files matching requirements
    class_diagram_match = 'inherit_graph*.png'

    ref = '.. _api_class_diagrams:\n\n'
    header = 'C++ Class diagrams'
    header += '\n' + len(header) * '=' + '\n\n'
    header = ref + header
    files = glob.glob(os.path.join(doxygen_path, class_diagram_match))
    realfiles = [os.path.join('../doxygen', os.path.basename(f)) for f in files]
    realfiles = ['/' + f for f in realfiles]
    outputfile = os.path.join(output_directory, 'class_diagrams.rst')
    with open(outputfile, 'w') as file:
        file.writelines(header)
        #params = [':height: 190 px', ':class: gallery']
        params = [':class: gallery']
        img_prefix = '.. image:: '

        for f in realfiles :
            line = img_prefix + f
            for p in params:
                line += '\n    ' + p
            line += '\n\n'
            file.writelines(line)


def filter_sphinx_warnings(warnfile):
    """Remove 'expected' warnings from sphinx logfile
    to keep only useful outputs
    
    """
    expected_warnings = [
        'WARNING: Duplicate declaration', # Known breathe warning
        'WARNING: Citation',
        'WARNING: image file not readable: ../inherit_graph',
        ]
    with open(warnfile) as ff:
        keep_msg = ff.readlines()
        
    for msg in expected_warnings:
        with open(warnfile) as ff:
            # remove useless lines
            current = [n for n in ff.readlines() if (not n.find(msg) > -1)]
        keep_msg = [msg for msg in current if msg in keep_msg]

    with open(warnfile, 'w') as ff:
        for msg in keep_msg:
            ff.write(msg)

def filter_xml_formulas(xmlfile):
    """Preprocess xml files to replace python escape characters (like \f, \a ...)
    in latex formulas
    that swig can not properly handle. See https://github.com/swig/swig/issues/889.

    Parameters
    ----------
    xmlfile: string
        xml file name (full path) (in-out param)
    """
    fileout = xmlfile.split('.')[0] + '.tmp'
    replace_sh = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'replace_py_escape.sh')
    subprocess.call([replace_sh, xmlfile, fileout])
    shutil.move(fileout, xmlfile)

def filter_dot_in_xml_formulas(xmlfile):
    """Replace \\dot with \dot in xml input.
    
    dot is confusing for doxygen (may be latex or graphviz), so in latex formula inside \rst 
    we need to use \\dot and replace it later, when breathe/sphinx comes into action.

    Parameters
    ----------
    xmlfile: string
        xml file name (full path) (in-out param)
    """
    fileout = xmlfile.split('.')[0] + '.tmp'
    with open(xmlfile, 'r+', encoding='utf8') as f:
        lines = f.read()
    newlines = lines.replace(r'\\dot', r'\dot')
    with open(fileout, 'w', encoding='utf8') as f:
        f.write(newlines)
    shutil.move(fileout, xmlfile)

def replace_latex(pyfile, latex_dir):
    """Post processing of latex forms in docstrings.
    
    Parameters
    ----------
    pyfile : string
       name (full path) of the python file to process
    latex_dir : string
       directory which contains pickle files with latex forms
       (result of do_latex or do_verbatim call in sicodoxy2swig)

    Usually : pyfile = some_component.py (e.g. numerics.py)
    and latex_dir = wrap/siconos/tmp_component_name.

    """
    # Parse latex_dir and get all pickle files
    formfiles = glob.glob(os.path.join(latex_dir, 'latex_*.pickle'))

    # Runner : use sed rather than python replace. But less portable (?)
    #runner = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'replace_latex.sh')

    # temp file for outputs.
    target = pyfile + '.copy'
    shutil.copyfile(pyfile, target)

    # Read input (.py)
    with open(pyfile, "r") as f:
        source_lines = f.readlines()
    
    rst = []
    # Parse and replace :
    # for each formula found in each pickle file,
    # replace FORMULA_Id with the proper string
    # in temp list.
    for fname in formfiles:
        with open(fname, 'rb') as f:
            latex_dict = pickle.load(f)
            for form in latex_dict:
                idf = 'FORMULA' + str(form) + '_'
                # we must \\dot in \rst doxygen
                # else there is a confusion with dot from graphviz.
                formula = latex_dict[form]["latex"].replace(r'\\', '\\\\')
                # escape \
                formula = latex_dict[form]["latex"].replace('\\', '\\\\')
                formula_type = latex_dict[form]["label"] # inline or not
                #formula = ''.join(formula)
                for line in source_lines:
                    if formula_type == 'inline':
                        rst.append(line.replace(idf, formula))
                    else:
                        indent = len(line) - len(line.lstrip())
                        rst.append(line.replace(idf, textwrap.indent(formula, indent * ' ')))
                source_lines = list(rst)
                rst = []
                #cmd = [runner, idf, formula, source, target]
                #print(cmd)
                #subprocess.call(cmd)
                #shutil.copyfile(target, source)


    # Replace .py with new results.
    with open(target, 'w') as f:
        for line in source_lines:
            f.write(line)
    shutil.move(target, pyfile)
