import os
import re
import unicodedata

from libshorttext.converter import *


def tokenizer(text):

    def trim_links(input_text):
        input_text = re.sub(r' (http|www|t\.co).*?(?: |$)', '', input_text)
        return input_text

    def trim_char(c):
        if ord(c) > 127:
            return ''
        if c.isdigit() or c.isalpha() or c.isspace():
            return c
        if re.match(r'[#$/\-_]', c):
            return c
        else:
            return ''

    def trim_hashtags(input_text):
        tag = re.search(r'#+([\w_]+[\w\'_\-]*[\w_]+)', input_text)
        if tag:
            hashtag = tag.group(0)
            tag_text = tag.group(1)
            splittable = re.match(r'.*([A-Z]|_|-|\d)', tag_text)
            if splittable:
                tokens = [token for token in re.split(r'([A-Z][a-z]*)|_|-|(\d*)', tag_text) if token]
                new_text = ' '.join(tokens)
                input_text = input_text.replace(hashtag, new_text)
                return trim_hashtags(input_text)
            else:
                input_text = input_text.replace(hashtag, tag_text)
                return trim_hashtags(input_text)
        else:
            return input_text

    def trim_digits(input_text):
        return re.sub(r'\d+', r'#', input_text)

    def trim_spaces(input_text):
        return re.sub(r'\s+', r' ', input_text)

    text = unicodedata.normalize('NFD', unicode(text, 'utf-8'))
    text = trim_links(text)
    text = ''.join(map(trim_char, text))
    text = trim_hashtags(text)
    text = re.sub(r'([a-z])([0-9])', r'\1 \2', text)
    text = re.sub(r'([0-9])([a-z])', r'\1 \2', text)
    text = trim_digits(text)
    text = trim_spaces(text)
    return text.strip().lower().split()

text_converter = Text2svmConverter('-stopword 1 -stemming 1')
text_converter.text_prep.tokenizer = tokenizer


def process():
    os.chdir(os.path.dirname(__file__))
    convert_text('training_data/post_sale,event,food,info', text_converter, 'outputs/test1.svm')
    text_converter.save('outputs/test1.text_converter')

process()

Add custom text2svm preprocessor with a blvd-minded tokenizer.
Add custom text2svm preprocessor with a blvd-minded tokenizer.
