'Return a tokenized copy of *text*, where each "token" represents
a separate topic.'

def tokenize(self, text):

    lowercase_text = text.lower()
    paragraph_breaks = self._mark_paragraph_breaks(text)
    text_length = len(lowercase_text)
    nopunct_text = ''.join((c for c in lowercase_text if re.match("[a-z\\-'    \n DCTB ]", c)))
    nopunct_par_breaks = self._mark_paragraph_breaks(nopunct_text)
    tokseqs = self._divide_to_tokensequences(nopunct_text)
    for ts in tokseqs:
        ts.wrdindex_list = [wi for wi in ts.wrdindex_list if (wi[0] not in self.stopwords)]
    token_table = self._create_token_table(tokseqs, nopunct_par_breaks)
    if (self.similarity_method == BLOCK_COMPARISON):
        gap_scores = self._block_comparison(tokseqs, token_table)
    elif (self.similarity_method == VOCABULARY_INTRODUCTION):
        raise NotImplementedError('Vocabulary    introduction    not    implemented')
    if (self.smoothing_method == DEFAULT_SMOOTHING):
        smooth_scores = self._smooth_scores(gap_scores)
    depth_scores = self._depth_scores(smooth_scores)
    segment_boundaries = self._identify_boundaries(depth_scores)
    normalized_boundaries = self._normalize_boundaries(text, segment_boundaries, paragraph_breaks)
    segmented_text = []
    prevb = 0
    for b in normalized_boundaries:
        if (b == 0):
            continue
        segmented_text.append(text[prevb:b])
        prevb = b
    if (prevb < text_length):
        segmented_text.append(text[prevb:])
    if (not segmented_text):
        segmented_text = [text]
    if self.demo_mode:
        return (gap_scores, smooth_scores, depth_scores, segment_boundaries)
    return segmented_text
