# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/elias-ela/amharic-tokenizer/blob/main/Amharic_Tokenizer.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + id="iyMFL8dK3StB" colab={"base_uri": "https://localhost:8080/"} outputId="3added65-6be9-4b5f-ea02-ff3431f927f5"
# !pip install sentencepiece

# + id="e0x1Zzn13YQW"
import sentencepiece as spm
import time

# + colab={"base_uri": "https://localhost:8080/"} id="ajTf5tl9Iuai" outputId="4265530f-e0b4-4623-a24c-5a6331f4235c"
from google.colab import drive
drive.mount('/content/drive/')

# + id="N6py_iBFAteN" colab={"base_uri": "https://localhost:8080/"} outputId="df7a86e6-fd87-430f-a776-df967abf9274"
dataPath = 'drive/MyDrive/Colab_Notebooks/am-mono'
print(dataPath)


# + [markdown] id="KPMLMlcxKhd0"
# # **Function that processes number of lines, blank lines, sentences, and words in a give file.**

# + id="D-aRN9g5DQyv"
def processFile(filename):
  lines, blanklines, sentences, words = 0, 0, 0, 0
  try:
    file = open(dataPath, 'r')
  except IOError:
    print(f"Unable to open file '%s'" % dataPath)
  
  for line in file:
    lines += 1

    if line.startswith('\n'):
      blanklines += 1
    else:
      # assuming that each sentence ends with :: or ?
      sentences += line.count('።') + line.count('?')

    tempwords = line.split(None)

    words += len(tempwords)

  file.close()

  print(f"-"*50)
  print(f"Lines         : {lines:,d}")
  print(f"Blank lines   : {blanklines:,d}")
  print(f"Sentences     : {sentences:,d}")
  print(f"Words         : {words:,d}")


# + colab={"base_uri": "https://localhost:8080/"} id="H4fAlF1wGWsV" outputId="a0416faf-8d94-4e35-c14a-126342d7cf6e"
processFile(dataPath)

# + [markdown] id="Ml5U20NDK7Th"
# # **Train sentencePiece for Amharic using BPE (Byte Pair Encoding)**

# + colab={"base_uri": "https://localhost:8080/"} id="Y8CcKZVZ6h-5" outputId="c19377a3-ea44-4a7e-d1b6-5be372ac6ada"
start_time = time.time()
spm.SentencePieceTrainer.train(input=dataPath, model_prefix='am-bpe', vocab_size=16000, model_type="bpe", character_coverage=1.0)
print("--- %s seconds ---" % (time.time() - start_time))

# + colab={"base_uri": "https://localhost:8080/"} id="7d1K4guC7dB2" outputId="f1413394-fd0b-46c4-85fc-c23e98548b3b"
model = spm.SentencePieceProcessor(model_file='am-bpe.model')
model.encode('ለአማርኛ ተናጋሪዎች የቀረበ መረጃ።', out_type=str)

# + [markdown] id="736gQDu3LkHg"
# # **Train sentencePiece for Amharic using Unigram.**

# + colab={"base_uri": "https://localhost:8080/"} id="-ro6Og0N_Z3t" outputId="a7a88f29-bc10-4c34-e649-8dd926c438d2"
start_time = time.time()
spm.SentencePieceTrainer.train(input=dataPath, model_prefix='am-unigram', vocab_size=16000, model_type="unigram", character_coverage=1.0)
print("--- %s seconds ---" % (time.time() - start_time))

# + colab={"base_uri": "https://localhost:8080/"} id="3LWP782zAX58" outputId="305c435b-60f2-4d63-b0e6-75be67691889"
model = spm.SentencePieceProcessor(model_file='am-unigram.model')
model.encode('ለአማርኛ ተናጋሪዎች የቀረበ መረጃ።', out_type=str)

# + [markdown] id="ta0__9mOLnYa"
# # **Train sentencePiece for Amharic using charcters**

# + colab={"base_uri": "https://localhost:8080/"} id="BZ8bOOfpBaUJ" outputId="b5ea48f9-3e6f-4fb2-9205-e979b5efcb48"
start_time = time.time()
spm.SentencePieceTrainer.train(input=dataPath, model_prefix='am-char', vocab_size=16000, model_type="char", character_coverage=1.0)
print("--- %s seconds ---" % (time.time() - start_time))

# + colab={"base_uri": "https://localhost:8080/"} id="T3epS8_TBaUL" outputId="32b3ffff-48a0-46e6-b928-c2a0fbcb45b7"
model = spm.SentencePieceProcessor(model_file='am-char.model')
model.encode('ለአማርኛ ተናጋሪዎች የቀረበ መረጃ።', out_type=str)

# + [markdown] id="gUAvPZTPLuYR"
# # **Train sentencePiece for Amharic using words**

# + colab={"base_uri": "https://localhost:8080/"} id="ETUjzrjDBxDk" outputId="e20f68de-2a14-49a3-9b79-dd7cbe713154"
start_time = time.time()
spm.SentencePieceTrainer.train(input=dataPath, model_prefix='am-word', vocab_size=16000, model_type="word", character_coverage=1.0)
print("--- %s seconds ---" % (time.time() - start_time))

# + colab={"base_uri": "https://localhost:8080/"} id="JaYURM8uBxDm" outputId="2c2b1602-381a-4200-a6d0-1a82eed29bd9"
model = spm.SentencePieceProcessor(model_file='am-word.model')
model.encode('ለአማርኛ ተናጋሪዎች የቀረበ መረጃ።', out_type=str)

StarcoderdataJupyterScriptsDedupFiltered