parsing with libclang : getting CXX_BASE_SPECIFIER cursors when base types unknown
Question: I'm writing a **documentation generator** and getting the include paths right
is hell so **I just skip entirely every includes** when I parse a file. I also
tune by hands all problematic defines or #ifdef blocks that would get skipped
because of the missing includes (and different command line versus production
build).

the problem I noticed is that:

    
    
    struct ComplexBuffer : IAnimatable
    {
    };
    

With `IAnimatable` is not declared (or is forward declared).  
I'm using the python binding of clang.cindex so I use get_children for
iteration: this result comes out:

    
    
    Found grammar element "IAnimatable" {CursorKind.CLASS_DECL} [line=37, col=8]
    Found grammar element "ComplexBuffer" {CursorKind.STRUCT_DECL} [line=39, col=9]
    

if I complete the base type:

    
    
    class IAnimatable {};
    
    struct ComplexBuffer : IAnimatable
    

I get a correct output:

    
    
    Found grammar element "IAnimatable" {CursorKind.CLASS_DECL} [line=37, col=8]
    Found grammar element "ComplexBuffer" {CursorKind.STRUCT_DECL} [line=39, col=9]
    Found grammar element "class IAnimatable" {CursorKind.CXX_BASE_SPECIFIER} [line=39, col=25]
    Found grammar element "class IAnimatable" {CursorKind.TYPE_REF} [line=39, col=25]
    

Exactly what I want because I can detect the inheritance list to put in the
documentation.

This problem only arises because I skip all the includes.

Maybe I can workaround this by reparsing the declaration line by hand ?

EDIT PS : my parsing python script for the sake of completion:

    
    
    import clang.cindex
    
    index = clang.cindex.Index.create()
    tu = index.parse(sys.argv[1], args=["-std=c++98"], options=clang.cindex.TranslationUnit.PARSE_SKIP_FUNCTION_BODIES)
    
    def printall_visitor(node):
        print 'Found grammar element "%s" {%s} [line=%s, col=%s]' % (node.displayname, node.kind, node.location.line, node.location.column)
    
    def visit(node, func):
        func(node)
        for c in node.get_children():
            visit(c, func)
    
    visit(tu.cursor, printall_visitor)
    


Answer: I'm going to answer this one myself, because the code I came up with can be
useful to future googlers.

In the end, I have coded the both methods that are supposed to work to
retreive the list of base classes in the inheritance list on a class
declaration line.

one using the AST cursor and one fully manual, coping as much as it can with
C++ complexity.

here is the whole result:

    
    
    #!/usr/bin/env python
    # -*- coding: utf-8 -*-
    '''
    Created on 2013/12/09
    
    @author: voddou
    '''
    
    import sys
    import re
    import clang.cindex
    import os
    import string
    
    class bcolors:
        HEADER = '\033[95m'
        OKBLUE = '\033[94m'
        CYAN = '\033[96m'
        OKGREEN = '\033[92m'
        WARNING = '\033[93m'
        FAIL = '\033[91m'
        ENDC = '\033[0m'
        MAGENTA = '\033[95m'
        GREY = '\033[90m'
    
        def disable(self):
            self.HEADER = ''
            self.OKBLUE = ''
            self.OKGREEN = ''
            self.WARNING = ''
            self.FAIL = ''
            self.ENDC = ''
            self.CYAN = ''
            self.MAGENTA = ''
            self.GREY = ''
    
    from contextlib import contextmanager
    
    @contextmanager
    def scopedColorizer(color):
        sys.stdout.write(color)
        yield
        sys.stdout.write(bcolors.ENDC)
    
    #clang.cindex.Config.set_library_file("C:/python27/DLLs/libclang.dll")
    
    src_filepath = sys.argv[1]
    src_basename = os.path.basename(src_filepath)
    
    parseeLines = file(src_filepath).readlines()
    
    def trim_all(astring):
        return "".join(astring.split()) 
    
    def has_token(line, token):
        trimed = trim_all(line)
        pos = string.find(trimed, token)
        return pos != -1
    
    def has_any_token(line, token_list):
        results = [has_token(line, t) for t in token_list]
        return any(results)
    
    def is_any(astring, some_strings):
        return any([x == astring for x in some_strings])
    
    def comment_out(line):
        return "//" + line
    
    # alter the original file to remove #inlude directives and protective ifdef blocks
    for i, l in enumerate(parseeLines):
        if has_token(l, "#include"):
            parseeLines[i] = comment_out(l)
        elif has_any_token(l, ["#ifdef", "#ifdefined", "#ifndef", "#if!defined", "#endif", "#elif", "#else"]):
            parseeLines[i] = comment_out(l)
    
    index = clang.cindex.Index.create()
    tu = index.parse(src_basename,
                     args=["-std=c++98"],
                     unsaved_files=[(src_basename, "".join(parseeLines))],
                     options=clang.cindex.TranslationUnit.PARSE_SKIP_FUNCTION_BODIES)
    
    print 'Translation unit:', tu.spelling, "\n"
    
    def gather_until(strlist, ifrom, endtokens):
        """make one string out of a list of strings, starting from a given index, until one token in endtokens is found.
        ex: gather_until(["foo", "toto", "bar", "kaz"], 1, ["r", "z"])
            will yield "totoba"
        """
        result = strlist[ifrom]
        nextline = ifrom + 1
        while not any([string.find(result, token) != -1 for token in endtokens]):
            result = result + strlist[nextline]
            nextline = nextline + 1
        nearest = result
        for t in endtokens:
            nearest = nearest.partition(t)[0]
        return nearest
    
    def strip_templates_parameters(declline):
        """remove any content between < >
        """
        res = ""
        nested = 0
        for c in declline:
            if c == '>':
                nested = nested - 1
            if nested == 0:
                res = res + c
            if c == '<':
                nested = nested + 1
        return res
    
    # thanks Markus Jarderot from Stackoverflow.com
    def comment_remover(text):
        def replacer(match):
            s = match.group(0)
            if s.startswith('/'):
                return ""
            else:
                return s
        pattern = re.compile(
            r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
            re.DOTALL | re.MULTILINE
        )
        return re.sub(pattern, replacer, text)
    
    def replace_any_of(haystack, list_of_candidates, by_what):
        for cand in list_of_candidates:
            haystack = string.replace(haystack, cand, by_what)
        return haystack
    
    cxx_keywords = ["class", "struct", "public", "private", "protected"]
    
    def clean_name(displayname):
        """remove namespace and type tags
        """
        r = displayname.rpartition("::")[2]
        r = replace_any_of(r, cxx_keywords, "")
        return r
    
    def find_parents_using_clang(node):
        l = []
        for c in node.get_children():
            if c.kind == clang.cindex.CursorKind.CXX_BASE_SPECIFIER:
                l.append(clean_name(c.displayname))
        return None if len(l) == 0 else l
    
    # syntax based custom parsing
    def find_parents_list(node):
        ideclline = node.location.line - 1
        declline = parseeLines[ideclline]
        with scopedColorizer(bcolors.WARNING):
            print "class decl line:", declline.strip()
        fulldecl = gather_until(parseeLines, ideclline, ["{", ";"])
        fulldecl = clean_name(fulldecl)
        fulldecl = trim_all(fulldecl)
        if string.find(fulldecl, ":") != -1:    # if inheritance exists on the declaration line
            baselist = fulldecl.partition(":")[2]
            res = strip_templates_parameters(baselist)  # because they are separated by commas, they would break the split(",")
            res = comment_remover(res)
            res = res.split(",")
            return res
        return None
    
    # documentation generator
    def make_htll_visitor(node):
        if (node.kind == clang.cindex.CursorKind.CLASS_DECL
            or node.kind == clang.cindex.CursorKind.STRUCT_DECL
            or node.kind == clang.cindex.CursorKind.CLASS_TEMPLATE):
    
            bases2 = find_parents_list(node)
            bases = find_parents_using_clang(node)
            if bases is not None:
                with scopedColorizer(bcolors.CYAN):
                    print "class clang list of bases:", str(bases)
    
            if bases2 is not None:
                with scopedColorizer(bcolors.MAGENTA):
                    print "class manual list of bases:", str(bases2)
    
    
    def visit(node, func):
        func(node)
        for c in node.get_children():
            visit(c, func)
    
    visit(tu.cursor, make_htll_visitor)
    
    with scopedColorizer(bcolors.OKGREEN):
        print "all over"
    

this code has allowed me to accept incomplete C++ translation units, correctly
parsing declarations such as this:

    
    
    struct ComplexBuffer
        : IAnimatable
        , Bugger,
    
            Mozafoka
    {
    };
    

coping also with these:

    
    
    struct AnimHandler : NonCopyable, IHandlerPrivateGetter< AnimHandler, AafHandler > // CRTP
    {
    ...
    };
    

giving me this output:

    
    
    class manual list of bases: ['NonCopyable', 'IHandlerPrivateGetter<>']
    

which is nice, the `clang` function version didn't return a single class in
the base list. Now it is forseeable to merge the result of both these
functions using a `set` to be on the safe side in case the manual parser would
miss something. However I'm thinking this could cause subtle duplications
because of the difference between `displayname` and my own parser.

But there you go, googlers, a nice clang python documentation generator
template that doesn't need full correctness of the build options and is pretty
fast because it totally ignores the `include`s statements.

nice day to all.

