def analyze(self, output_folder=".", auto_remove=False):
        """
        :type auto_remove: boolean
        :param boolean auto_remove: auto remove previous files in analyze folder
        """
        if auto_remove:
            try:
                shutil.rmtree(output_folder)
            except:
                pass
        try:
            mkdir(output_folder)
        except:
            pass
        tokens = [token for sublist in self.sentences for token in sublist]
        df = pd.DataFrame(tokens)
        log = u""
        log += u"Sentences    : {}\n".format(len(self.sentences))
        n = df.shape[1]
        log += self._analyze_first_token(df, 0, output_folder)
        for i in range(1, n):
            log += self._analyze_field(df, i, output_folder)
        print(log)
        stat_file = join(output_folder, "stats.txt")
        write(stat_file, log)
def analyze(self, output_folder=".", auto_remove=False):
        """
        :type auto_remove: boolean
        :param boolean auto_remove: auto remove previous files in analyze folder
        """
        if auto_remove:
            try:
                shutil.rmtree(output_folder)
            except:
                pass
        try:
            mkdir(output_folder)
        except:
            pass
        tokens = [token for sublist in self.sentences for token in sublist]
        df = pd.DataFrame(tokens)
        log = u""
        log += u"Sentences    : {}\n".format(len(self.sentences))
        n = df.shape[1]
        log += self._analyze_first_token(df, 0, output_folder)
        for i in range(1, n):
            log += self._analyze_field(df, i, output_folder)
        print(log)
        stat_file = join(output_folder, "stats.txt")
        write(stat_file, log)
def analyze ( self , output_folder = "." , auto_remove = False ) : if auto_remove : try : shutil . rmtree ( output_folder ) except : pass try : mkdir ( output_folder ) except : pass tokens = [ token for sublist in self . sentences for token in sublist ] df = pd . DataFrame ( tokens ) log = u"" log += u"Sentences    : {}\n" . format ( len ( self . sentences ) ) n = df . shape [ 1 ] log += self . _analyze_first_token ( df , 0 , output_folder ) for i in range ( 1 , n ) : log += self . _analyze_field ( df , i , output_folder ) print ( log ) stat_file = join ( output_folder , "stats.txt" ) write ( stat_file , log )
:type auto_remove: boolean
        :param boolean auto_remove: auto remove previous files in analyze folder
: type auto_remove : boolean : param boolean auto_remove : auto remove previous files in analyze folder