# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# <img src="../Pics/MLSb-T.png" width="160">
# <br><br>
# <center><u><H1>Sentence splitter and Tokenization</H1></u></center>

# ## Sentence splitter:

string = 'Every one of us is, in the cosmic perspective, precious. If a human disagrees with you, let him live. In a hundred billion galaxies, you will not find another.'

from nltk.tokenize import sent_tokenize

_sent = sent_tokenize(string)

print(_sent)

import nltk.tokenize.punkt
#This tokenizer divides a text into a list of sentences,
#by using an unsupervised algorithm to build a model for abbreviation
#words, collocations, and words that start sentences. 

tokenizer = nltk.tokenize.punkt.PunktSentenceTokenizer()

tokenizer.tokenize(string)

# ## Tokenization:

# ### Word Tokenizing:

a = 'Hi NLTK students ! level s10'

# simplest tokenizer: uses white space as delimiter.
print(a.split())

from nltk.tokenize import word_tokenize

word_tokenize(a)

# Another method using TreebankWorldTokenizer
from nltk.tokenize import TreebankWordTokenizer

tokenizer = TreebankWordTokenizer()
print(tokenizer.tokenize(a))

# ### Removing Noise

# +
import re
# Example of removing numbers:

def remove_numbers(text):
    return re.sub(r'\d+', "", text)


# -

txt = 'This a sample sentence in English, \n with   whitespaces and many numbers 123456!'

print('Removed numbers:', remove_numbers(txt))

# +
# example of removing punctuation from text
import string

def remove_punctuation(text):
    words = word_tokenize(text)
    pun_removed = [w for w in words if w.lower() not in string.punctuation]
    return " ".join(pun_removed)


# -

b = 'This a great course of NLP using Python and NLTK!!! for this year 2017, isnt.?'
print(remove_punctuation(b))

from nltk.tokenize import regexp_tokenize, wordpunct_tokenize, blankline_tokenize

# # + : one or more times | \w : character or digit
regexp_tokenize(b, pattern='\w+')

regexp_tokenize(b, pattern='\d+')

c = 'The capital is raising up to $100000'

regexp_tokenize(c, pattern='\w+|\$')

# # + = one or more times
regexp_tokenize(c, pattern='\w+|\$[\d]')

regexp_tokenize(c, pattern='\w+|\$[\d\.]+|\S+')

wordpunct_tokenize(b)

blankline_tokenize(b)

# ## References:
#
# https://docs.python.org/2/library/tokenize.html
#
# http://www.nltk.org/_modules/nltk/tokenize.html
#
# http://www.nltk.org/_modules/nltk/tokenize/punkt.html
#
# http://www.nltk.org/_modules/nltk/tokenize/treebank.html
#
# http://www.nltk.org/_modules/nltk/tokenize/regexp.html



StarcoderdataJupyterScriptsDedupFiltered