'Python port of the Moses detokenizer.
:param tokens: A list of strings, i.e. tokenized text.
:type tokens: list(str)
:return: str'

def tokenize(self, tokens, return_str=False):

    text = u'    {}    '.format('    '.join(tokens))
    text = text_type(text)
    (regexp, substitution) = self.AGGRESSIVE_HYPHEN_SPLIT
    text = re.sub(regexp, substitution, text)
    text = self.unescape_xml(text)
    quote_counts = {u"'": 0, u'"': 0, u'``': 0, u'`': 0, u"''": 0}
    prepend_space = '    '
    detokenized_text = ''
    tokens = text.split()
    for (i, token) in enumerate(iter(tokens)):
        if is_cjk(token[0]):
            if ((i > 0) and is_cjk(token[(-1)])):
                detokenized_text += token
            else:
                detokenized_text += (prepend_space + token)
            prepend_space = '    '
        elif (token in self.IsSc):
            detokenized_text += (prepend_space + token)
            prepend_space = ''
        elif re.search('^[\\,\\.\\?\\!\\:\\;\\\\\\%\\}\\]\\)]+$', token):
            if ((self.lang == 'fr') and re.search('^[\\?\\!\\:\\;\\\\\\%]$', token)):
                detokenized_text += '    '
            detokenized_text += token
            prepend_space = '    '
        elif ((self.lang == 'en') and (i > 0) and re.search(u"^['][{}]".format(self.IsAlpha), token)):
            detokenized_text += token
            prepend_space = '    '
        elif ((self.lang == 'cs') and (i > 1) and re.search('^[0-9]+$', tokens[(-2)]) and re.search('^[.,]$', tokens[(-1)]) and re.search('^[0-9]+$', token)):
            detokenized_text += token
            prepend_space = '    '
        elif ((self.lang in ['fr', 'it']) and (i <= (len(tokens) - 2)) and re.search(u"[{}][']$".format(self.IsAlpha), token) and re.search(u'^[{}]$'.format(self.IsAlpha), tokens[(i + 1)])):
            detokenized_text += (prepend_space + token)
            prepend_space = ''
        elif ((self.lang == 'cs') and (i <= (len(tokens) - 3)) and re.search(u"[{}][']$".format(self.IsAlpha), token) and re.search(u'^[-\u2013]$', tokens[(i + 1)]) and re.search(u'^li$|^mail.*', tokens[(i + 2)], re.IGNORECASE)):
            detokenized_text += ((prepend_space + token) + tokens[(i + 1)])
            next(tokens, None)
            prepend_space = ''
        elif re.search('^[\\\'\\"\xe2\x80\x9e\xe2\x80\x9c`]+$', token):
            normalized_quo = token
            if re.search('^[\xe2\x80\x9e\xe2\x80\x9c\xe2\x80\x9d]+$', token):
                normalized_quo = '"'
            quote_counts[normalized_quo] = quote_counts.get(normalized_quo, 0)
            if ((self.lang == 'cs') and (token == u'\u201e')):
                quote_counts[normalized_quo] = 0
            if ((self.lang == 'cs') and (token == u'\u201c')):
                quote_counts[normalized_quo] = 1
            if ((quote_counts[normalized_quo] % 2) == 0):
                if ((self.lang == 'en') and (token == u"'") and (i > 0) and re.search('[s]$', tokens[(i - 1)])):
                    detokenized_text += token
                    prepend_space = '    '
                else:
                    detokenized_text += (prepend_space + token)
                    prepend_space = ''
                    quote_counts[normalized_quo] += 1
            else:
                detokenized_text += token
                prepend_space = '    '
                quote_counts[normalized_quo] += 1
        elif ((self.lang == 'fi') and re.search(':$', tokens[(i - 1)]) and re.search(self.FINNISH_REGEX, token)):
            detokenized_text += (prepend_space + token)
            prepend_space = '    '
        else:
            detokenized_text += (prepend_space + token)
            prepend_space = '    '
    (regexp, substitution) = self.ONE_SPACE
    detokenized_text = re.sub(regexp, substitution, detokenized_text)
    detokenized_text = detokenized_text.strip()
    return (detokenized_text if return_str else detokenized_text.split())
