Segmentation fault while using a C module in a .so file
Question: I got a segmentation fault – probably caused by the long loop – while running
my python script at the command line under Linux. I know exactly _where_ the
problem is, but I don't know _why_. I've tried some of the method I searched
online, including this site, but I still cannot solve it. So, please help me –
thank you in advance. Here follows some of the code:

### 0

`analyzer.py`, where the program begins:

    
    
    from classify import BayesClassifier
    class Analyzer:
    
        def __init__(self):
            self.classify = BayesClassifier('/home/user/yakamoz/srcanalyzer/classification/training.tab')
    
    if __name__ == '__main__':
        a = Analyzer()
        # the following is a string of Chinese character, which, I am sure,
        # has no influence on the Segmentation fault, you can just suppose
        # it as a paragraph in English.
    
        text = "市委常委、纪委书记杨娟高度评价我县基层党风廉政建设：\
           务实创新成效显著作者：县纪委办公室发布时间：11月27日下午，\
           市委常委、纪委书记杨娟率领市纪委副书记蒋玉平、王友富，\
           市纪委常委、秘书长任斌，市纪委机关党委书记刘林建一行来我\
           县调研基层党风廉政建设。调研中，杨娟高度评价我县基层党风廉政建设，\
           认为工作务实创新，成效显著。县委书记陈朝先，县委副书记季代双，县委常委、\
           纪委书记韩忠明陪同调研。杨娟一行先后来到两河镇、西部花都、两江广场、\
           工业园区等地实地调研我县基层党风廉政建设，检阅我县“两化”互动、“三化”\
           联动发展成果。查阅相关资料在两河镇，杨娟认真听取了两河片区纪工委\
           日常工作开展情况的汇报，仔细翻阅了巡查工作日记和接访记录。杨娟指出，\
           设置乡镇片区纪工委是加强基层纪检组织建设的创新举措。\
           盐亭在全市率先设置、运行纪工委以来，在化解农村信访矛盾，理顺群众情绪，\
           强化基层办案工作等方面取得了明显成效。她要求，要总结提炼片区纪工委的经验，\
           进一步明确职能职责，在机构设置、人员配备、制度建设等方面进行探索实践，\
           为全市基层纪检组织建设提供有益经验借鉴。杨娟还饶有兴趣地参观了两河镇\
           的机关廉政文化建设"
    
        print str(a.classify.classify_text(text)[0])
    

### 1

`classify.py`; this file is used by the `analyzer.py`, presented above:

    
    
    # -*- coding:utf-8 -*-
    from match import WordMatch
    import cPickle
    import math
    
    class BayesClassifier:
    
        __trainingdata = {}                    
        __classifywordscount = {}
        __classifydoccount = {}
    
        def __init__(self, table_name):         
            self.trainingtable = cPickle.load(open(table_name, 'r'))  
            for x in self.trainingtable:
                self.train(x[1], x[0])  
            print 'training finished'
            self.matrix = self.get_matrix()         
            self.vector_count = len(self.matrix)
            self.doc_count = len(self.trainingtable)
            self.match = WordMatch(self.matrix)
    
        def get_matrix(self):                       
            matrix = {}
            for x in self.trainingtable:
                for k in x[0]:
                    matrix[k] = 0
            return matrix
    
        def doc_to_vector(self, content):
            matrix = {word:value for (word, value) in self.match.find(content).items()}     
            return matrix        
    
        def train(self, cls, vector):
            if cls not in self.__trainingdata:
                self.__trainingdata[cls] = {}
            if cls not in self.__classifywordscount:
                self.__classifywordscount[cls] = 0
            if cls not in self.__classifydoccount:
                self.__classifydoccount[cls] = 0
            self.__classifydoccount[cls] += 1
    
            for word in vector.keys():
                self.__classifywordscount[cls] += vector[word]
                if word not in self.__trainingdata[cls]:
                    self.__trainingdata[cls][word] = vector[word]
                else:
                    self.__trainingdata[cls][word] += vector[word]
    
    
        def classify_text(self, content):
            t = -1 << 32
            res = "unknown classification"
            for cls in self.__trainingdata.keys():
                prob = self.__count(cls, self.doc_to_vector(content))
                if prob > t:
                    res = cls
                    t = prob
            return res, t
    

### 2

`match.py`; this code is referenced by the `classify.py`

    
    
    # -*- coding:utf-8 -*-
    import os
    import re
    import util.ahocorasick.x64 as ahocorasick
    # util.ahocorasick.x64 is a folder where .so file locates
    
    class WordMatch(object):
        def __init__(self, arg):
            self.__tree = ahocorasick.KeywordTree()
            if isinstance(arg, (list, dict)):
                for item in arg:
                    if item:
                        self.__tree.add(item)
            elif isinstance(arg, basestring):
                if os.path.isfile(arg):
                    fp = open(arg)
                    for line in fp:
                        line = line.strip()
                        if line:
                            self.__tree.add(line)
                    fp.close()
                else:
                    print 'the path of the input file does not exist'
                    return
            else:
                print 'parameter fault'
                return           
            self.__tree.make()
    
        def _findall(self, content):
            '''return the list of keywords that is found 
            '''
            hit_list = []
            if isinstance(content, basestring):
                for start, end in self.__tree.findall(content):
                    if len(content[start:end]):
                        hit_list.append(content[start:end])
            else:
                print 'AC automation requires string '
            return hit_list
    
        def find(self, content):
            '''return those matched keywords and the corresponding count
            '''
            hit_list = self._findall(content)
            mydict = {}
            for item in hit_list:
                if item in mydict:
                    mydict[item] += 1
                else:
                    mydict[item] = 1
            return mydict
    

### 3

`__init__.py`, under the folder `util.ahocorasick.x64`:

    
    
    import _ahocorasick
    
    __all__ = ['KeywordTree']
    
    
    # A high level version of the keyword tree.  Most of the methods here
    # are just delegated over to the underlying C KeywordTree 
    #(in the .so file, which is not shown here).
    
    
    class KeywordTree(object):
        def __init__(self):
            self.__tree = _ahocorasick.KeywordTree();
    
    
        def add(self, s):
            return self.__tree.add(s)
    
    
        def make(self):
            return self.__tree.make()
    
    
        def zerostate(self):
            return self.__tree.zerostate()
    
        ##### !! I found this is where the segmentation fault occurs
    
        def __findall_helper(self, sourceBlock, allow_overlaps, search_function):
            """Helper function that captures the common logic behind the
            two findall methods."""
            startpos = 0
            startstate = self.zerostate()
            loop_times = 0            
    
            while True:
                #print spot_1
                match = search_function(sourceBlock, startpos, startstate)
                #print spot_2
                if not match:
                    break
                yield match[0:2]
                startpos = match[1]
                if allow_overlaps: #which in my case is always false
                    startstate = match[2]
                else:
                    loop_times = loop_times + 1
                    #print spot_3
                    startstate = self.zerostate()
                    #print spot_4
                    #print loop_times
    
        def findall(self, sourceBlock, allow_overlaps=0):
            return self.__findall_helper(sourceBlock, allow_overlaps,self.__tree.search)
    

I am confused by the different result given: I've found that the problem lies
in 3 `__init__.py` or rather, the `__findall_helper(self, sourceBlock,
akkow_overlaps, search_function)`.

By uncommenting one of the following annotation:

    
    
    #print spot_1
    #print spot_2
    #print spot_4
    

one can eliminate the segmentation fault and the loop is finite (the match can
be `None`), but by uncommenting the `#print spot_3`, one can _not_ (it seems
like an infinite loop). Here comes my question:

Does the `print` statement has a side effect in python? I found only that
there is a `print` statement in one of the three spots mentioned above
(`spot_1` or `spot_2` or `spot_4`) can eliminate the fault. By the way, I
found this by accident, there are no `print` at first.

### 4

Here is the `backtrace` using `gdb`.

    
    
    (gdb) r analyzer.py
    
    Starting program: /usr/local/bin/python analyzer.py
    [Thread debugging using libthread_db enabled]
    Detaching after fork from child process 11499.
    training finished
    
    Program received signal SIGSEGV, Segmentation fault.
    0x00007ffff178956d in ahocorasick_KeywordTree_search_helper (state=0x85c730, 
    string=0x8967d4 "【中国环保在线 市场行情】“我国将在2016年启动全国碳市场。全国碳交    易市场的首批行业企业将由电力、冶金、有色、建材、化工5个传统制造业和航", <incomplete     sequence \347\251>..., n=140733193395828, startpos=118366835, 
    out_start=0x7fffffffd468, out_end=0x7fffffffd460,       out_last_state=0x7fffffffd458) at aho-corasick.c:216
    216 aho-corasick.c: No such file or directory.
    in aho-corasick.c
    Missing separate debuginfos, use: debuginfo-install glibc-2.12-    1.149.el6.x86_64
    
    (gdb) bt
    
    #0  0x00007ffff178956d in ahocorasick_KeywordTree_search_helper     (state=0x85c730, string=0x8967d4 "【中国环保在线 市场行情】“我国将在2016年启动全国碳市场。全国碳交     易市场的首批行业企业将由电力、冶金、有色、建材、化工5个传统制造业和航", <incomplete sequence \347\251>..., n=140733193395828, startpos=118366835, out_start=0x7fffffffd468, out_end=0x7fffffffd460,   out_last_state=0x7fffffffd458) at aho-corasick.c:216
    #1  0x00007ffff178a2b1 in ahocorasick_KeywordTree_basesearch     (self=0x7ffff7f6c230, args=0x7ffff0ca1a50, kwargs=0x0, helper=0x7ffff1789525<ahocorasick_KeywordTree_search_helper>) at     py_wrapper.c:190
    #2  0x00007ffff178a358 in ahocorasick_KeywordTree_search  (self=0x7ffff7f6c230, args=0x7ffff0ca1a50, kwargs=0x0) at py_wrapper.c:212
    #3  0x00000000004a7103 in call_function (f=<value optimized out>, throwflag=<value optimized out>) at Python/ceval.c:4013
    #4  PyEval_EvalFrameEx (f=<value optimized out>, throwflag=<value optimized out>) at Python/ceval.c:2666
    #5  0x0000000000507e8d in gen_send_ex (gen=0x7904640, arg=0x0, exc=<value optimized out>) at Objects/genobject.c:84
    #6  0x00000000004a25da in PyEval_EvalFrameEx (f=<value optimized out>, throwflag=<value optimized out>) at Python/ceval.c:2497
    #7  0x00000000004a805b in fast_function (f=<value optimized out>, throwflag=<value optimized out>) at Python/ceval.c:4099
    #8  call_function (f=<value optimized out>, throwflag=<value optimized out>) at Python/ceval.c:4034
    #9  PyEval_EvalFrameEx (f=<value optimized out>, throwflag=<value optimized out>) at Python/ceval.c:2666
    #10 0x00000000004a8bd7 in PyEval_EvalCodeEx (co=0x7ffff1ff54b0, globals=<value optimized out>, locals=<value optimized out>, args=<value optimized out>, argcount=3, kws=0x9984520, kwcount=0, defs=0x7ffff2016968, defcount=1, closure=0x0) at Python/ceval.c:3253
    #11 0x00000000004a6dce in fast_function (f=<value optimized out>, throwflag=<value optimized out>) at Python/ceval.c:4109
    #12 call_function (f=<value optimized out>, throwflag=<value optimized out>) at Python/ceval.c:4034
    #13 PyEval_EvalFrameEx (f=<value optimized out>, throwflag=<value optimized out>) at Python/ceval.c:2666
    #14 0x00000000004a805b in fast_function (f=<value optimized out>, throwflag=<value optimized out>) at Python/ceval.c:4099
    #15 call_function (f=<value optimized out>, throwflag=<value optimized out>) at Python/ceval.c:4034
    #16 PyEval_EvalFrameEx (f=<value optimized out>, throwflag=<value optimized out>) at Python/ceval.c:2666
    #17 0x00000000004a8bd7 in PyEval_EvalCodeEx (co=0x7ffff7ec4130, globals=<value optimized out>, locals=<value optimized out>, args=<value optimized out>, argcount=0, kws=0x0, kwcount=0, defs=0x0, defcount=0, closure=0x0) at Python/ceval.c:3253
    #18 0x00000000004a8ce2 in PyEval_EvalCode (co=<value optimized out>, globals=<value optimized out>, locals=<value optimized out>) at Python/ceval.c:667
    #19 0x00000000004c91fe in run_mod (fp=0x880ee0, filename=0x7fffffffe30c "analyzer.py", start=<value optimized out>, globals=0x7fc140, locals=0x7fc140, closeit=1, flags=0x7fffffffdea0) at Python/pythonrun.c:1346
    #20 PyRun_FileExFlags (fp=0x880ee0, filename=0x7fffffffe30c "analyzer.py", start=<value optimized out>, globals=0x7fc140, locals=0x7fc140, closeit=1,flags=0x7fffffffdea0) at Python/pythonrun.c:1332
    #21 0x00000000004c9414 in PyRun_SimpleFileExFlags (fp=0x880ee0, filename=0x7fffffffe30c "analyzer.py", closeit=1, flags=0x7fffffffdea0)at Python/pythonrun.c:936
    #22 0x0000000000414a4f in Py_Main (argc=<value optimized out>, argv=<value optimized out>) at Modules/main.c:599
    #23 0x0000003fd281ed5d in __libc_start_main () from /lib64/libc.so.6
    #24 0x0000000000413bc9 in _start ()
    


Answer: I see

    
    
      self.__tree = _ahocorasick.KeywordTree();
    

Then

    
    
    self.__tree.zerostate()
    

and finally

    
    
    return self.__findall_helper(sourceBlock, allow_overlaps,self.__tree.search_long)
    

So my guess is that the function `search_long` is invalidated when you do
`__tree.zerostate()` so you get undefined behaviour which in some cases leads
to a segfault. It's a lot of code and there's a opaque library so it's hard to
tell for sure. Best thing is to go to the documentation and make sure you're
using the library correctly.

The `print` is a red herring, and by allocating something just forces the
crash to happen sooner.

Hope it helps.

