examples/arabic_sts/arabic_preprocess.py
def preprocess(text, do_farasa_tokenization=True, farasa=None, use_farasapy=False):
    """
	Preprocess takes an input text line an applies the same preprocessing used in araBERT
				pretraining

	Note: a farasapy segmenter is ~6x faster than the py4j.java_gateway, consider setting use_farasapy=True
	Farsa Segmentation will soon be fully migrated to farasapy, and support for the py4j.java_gateway.JavaObject will be removed
	Args:
		text (string): inout text string
		farasa (JavaGateway): pass a "py4j.java_gateway.JavaObject" to the FarasaSegmenter.jar file
		farasa (FarasaSegmenter): pass a FarasaSegmenter instance from farasapy.segmenter
		use_farasapy (boolean): set it to True when using a FarasaSegmenter instance from farasapy.segmenter
	Example:
		from py4j.java_gateway import JavaGateway
		gateway = JavaGateway.launch_gateway(classpath='./FarasaSegmenterJar.jar')
		farasa = gateway.jvm.com.qcri.farasa.segmenter.Farasa()
		processed_text = preprocess("Some_Text",do_farasa_tokenization=True , farasa=farasa)
	"""
    text = str(text)
    processing_text = araby.strip_tashkeel(text)
    processing_text = re.sub(r"\d+\/[ء-ي]+\/\d+\]", "", processing_text)
    processing_text = re.sub("ـ", "", processing_text)
    processing_text = re.sub("[«»]", ' " ', processing_text)
    # replace the [رابط] token with space if you want to clean links
    processing_text = re.sub(regex_url_step1, "[رابط]", processing_text)
    processing_text = re.sub(regex_url_step2, "[رابط]", processing_text)
    processing_text = re.sub(regex_url, "[رابط]", processing_text)
    processing_text = re.sub(regex_email, "[بريد]", processing_text)
    processing_text = re.sub(regex_mention, "[مستخدم]", processing_text)
    processing_text = re.sub("…", r"\.", processing_text).strip()
    processing_text = remove_redundant_punct(processing_text)

    processing_text = re.sub(
        r"\[ رابط \]|\[ رابط\]|\[رابط \]", " [رابط] ", processing_text
    )
    processing_text = re.sub(
        r"\[ بريد \]|\[ بريد\]|\[بريد \]", " [بريد] ", processing_text
    )
    processing_text = re.sub(
        r"\[ مستخدم \]|\[ مستخدم\]|\[مستخدم \]", " [مستخدم] ", processing_text
    )

    processing_text = remove_elongation(processing_text)
    if do_farasa_tokenization and farasa is not None:
        processing_text = tokenize_arabic_words_farasa(
            processing_text, farasa, use_farasapy
        )
    return processing_text.strip()

	Preprocess takes an input text line an applies the same preprocessing used in araBERT
				pretraining

	Note: a farasapy segmenter is ~6x faster than the py4j.java_gateway, consider setting use_farasapy=True
	Farsa Segmentation will soon be fully migrated to farasapy, and support for the py4j.java_gateway.JavaObject will be removed
	Args:
		text (string): inout text string
		farasa (JavaGateway): pass a "py4j.java_gateway.JavaObject" to the FarasaSegmenter.jar file
		farasa (FarasaSegmenter): pass a FarasaSegmenter instance from farasapy.segmenter
		use_farasapy (boolean): set it to True when using a FarasaSegmenter instance from farasapy.segmenter
	Example:
		from py4j.java_gateway import JavaGateway
		gateway = JavaGateway.launch_gateway(classpath='./FarasaSegmenterJar.jar')
		farasa = gateway.jvm.com.qcri.farasa.segmenter.Farasa()
		processed_text = preprocess("Some_Text",do_farasa_tokenization=True , farasa=farasa)
	
Preprocess takes an input text line an applies the same preprocessing used in araBERT
pretraining

a farasapy segmenter is ~6x faster than the py4j.java_gateway, consider setting use_farasapy=True
Farsa Segmentation will soon be fully migrated to farasapy, and support for the py4j.java_gateway.JavaObject will be removed
Preprocess takes an input text line an applies the same preprocessing used in araBERT pretraining a farasapy segmenter is ~6x faster than the py4j . java_gateway consider setting use_farasapy = True Farsa Segmentation will soon be fully migrated to farasapy and support for the py4j . java_gateway . JavaObject will be removed
def preprocess(text, do_farasa_tokenization=True, farasa=None, use_farasapy=False):
    text = str(text)
    processing_text = araby.strip_tashkeel(text)
    processing_text = re.sub(r"\d+\/[ء-ي]+\/\d+\]", "", processing_text)
    processing_text = re.sub("ـ", "", processing_text)
    processing_text = re.sub("[«»]", ' " ', processing_text)
    processing_text = re.sub(regex_url_step1, "[رابط]", processing_text)
    processing_text = re.sub(regex_url_step2, "[رابط]", processing_text)
    processing_text = re.sub(regex_url, "[رابط]", processing_text)
    processing_text = re.sub(regex_email, "[بريد]", processing_text)
    processing_text = re.sub(regex_mention, "[مستخدم]", processing_text)
    processing_text = re.sub("…", r"\.", processing_text).strip()
    processing_text = remove_redundant_punct(processing_text)
    processing_text = re.sub(
        r"\[ رابط \]|\[ رابط\]|\[رابط \]", " [رابط] ", processing_text
    )
    processing_text = re.sub(
        r"\[ بريد \]|\[ بريد\]|\[بريد \]", " [بريد] ", processing_text
    )
    processing_text = re.sub(
        r"\[ مستخدم \]|\[ مستخدم\]|\[مستخدم \]", " [مستخدم] ", processing_text
    )
    processing_text = remove_elongation(processing_text)
    if do_farasa_tokenization and farasa is not None:
        processing_text = tokenize_arabic_words_farasa(
            processing_text, farasa, use_farasapy
        )
    return processing_text.strip()
def preprocess ( text , do_farasa_tokenization = True , farasa = None , use_farasapy = False ) : text = str ( text ) processing_text = araby . strip_tashkeel ( text ) processing_text = re . sub ( r"\d+\/[ء-ي]+\/\d+\]",  " ,  p ocessing_text) processing_text = re . sub ( "ـ", ", rocessing_text) processing_text = re . sub ( "[«»]",  ' " ',  p ocessing_text) processing_text = re . sub ( regex_url_step1 , "[رابط]", pr o essing_text) processing_text = re . sub ( regex_url_step2 , "[رابط]", pr o essing_text) processing_text = re . sub ( regex_url , "[رابط]", pr o essing_text) processing_text = re . sub ( regex_email , "[بريد]", pr o essing_text) processing_text = re . sub ( regex_mention , "[مستخدم]", proc e sing_text) processing_text = re . sub ( "…",  r \.",  p ocessing_text). s t rip() processing_text = remove_redundant_punct ( processing_text ) processing_text = re . sub ( r"\[ رابط \]|\[ رابط\]|\[رابط \]", " [رابط] " , processing_tex t ) processing_text = re . sub ( r"\[ بريد \]|\[ بريد\]|\[بريد \]", " [بريد] " , processing_tex t ) processing_text = re . sub ( r"\[ مستخدم \]|\[ مستخدم\]|\[مستخدم \]", " [مستخدم] ", pr o essing_text ) processing_text = remove_elongation ( processing_text ) if do_farasa_tokenization and farasa is not None : processing_text = tokenize_arabic_words_farasa ( processing_text , farasa , use_farasapy ) return processing_text . strip ( )
Preprocess takes an input text line an applies the same preprocessing used in araBERT
pretraining
Preprocess takes an input text line an applies the same preprocessing used in araBERT pretraining
"""
	Preprocess takes an input text line an applies the same preprocessing used in araBERT
				pretraining

	Note: a farasapy segmenter is ~6x faster than the py4j.java_gateway, consider setting use_farasapy=True
	Farsa Segmentation will soon be fully migrated to farasapy, and support for the py4j.java_gateway.JavaObject will be removed
	Args:
		text (string): inout text string
		farasa (JavaGateway): pass a "py4j.java_gateway.JavaObject" to the FarasaSegmenter.jar file
		farasa (FarasaSegmenter): pass a FarasaSegmenter instance from farasapy.segmenter
		use_farasapy (boolean): set it to True when using a FarasaSegmenter instance from farasapy.segmenter
	Example:
		from py4j.java_gateway import JavaGateway
		gateway = JavaGateway.launch_gateway(classpath='./FarasaSegmenterJar.jar')
		farasa = gateway.jvm.com.qcri.farasa.segmenter.Farasa()
		processed_text = preprocess("Some_Text",do_farasa_tokenization=True , farasa=farasa)
	""" # replace the [رابط] token with space if you want to clean links
text
inout text string
inout text string
do_farasa_tokenization
farasa
pass a FarasaSegmenter instance from farasapy.segmenter
pass a FarasaSegmenter instance from farasapy . segmenter
use_farasapy
set it to True when using a FarasaSegmenter instance from farasapy.segmenter
set it to True when using a FarasaSegmenter instance from farasapy . segmenter
examples
from py4j.java_gateway import JavaGateway
gateway = JavaGateway.launch_gateway(classpath='./FarasaSegmenterJar.jar')
farasa = gateway.jvm.com.qcri.farasa.segmenter.Farasa()
processed_text = preprocess("Some_Text",do_farasa_tokenization=True , farasa=farasa)
from py4j . java_gateway import JavaGateway gateway = JavaGateway . launch_gateway ( classpath = ' . / FarasaSegmenterJar . jar ' ) farasa = gateway . jvm . com . qcri . farasa . segmenter . Farasa () processed_text = preprocess ( " Some_Text " do_farasa_tokenization = True farasa = farasa )