from IPython.display import Image
Image(filename='entity_extraction_process.png')
# Note: this image is taken from NLTK Book and requires citation

"""
Explanation: Entity Extraction from Contract Documents
Problem Statement:
Many companies deal with the understanding of large amount text data in the documents. With the help of natural language processing one can break the down problem in to simple steps as shown below.
Display: Jupyter notebook
Language used: Python
Libraries used: nltk, re
Tasks accomplished:
1) Extracting addresses which are in various format from the given document
2) Tokenizing the text into sentences, words and then tagging with Name Entities. The process performed here is shown in the Figure 1
Note: This project is purely for the demonstration purpose and not to disclose any confidential data/procedures of any originazation
End of explanation
"""


# Importing NLTK Dependencies
import nltk, re
from nltk import word_tokenize, pos_tag, ne_chunk
from nltk.tokenize.punkt import PunktSentenceTokenizer


"""
Explanation: Figure 1: Entity Extraction Process
Analysis
End of explanation
"""


# using docx2txt libarary to avoid the encoding errors while reading the text in given word file
import docx2txt
raw_data = docx2txt.process('Contract_Template.docx')

"""
Explanation: Loading the data
End of explanation
"""


paragraphs = [p for p in raw_data.split('\n') if p]
tokenizer = PunktSentenceTokenizer()
sentences = [tokenizer.tokenize(paragraph) for paragraph in paragraphs]
sentences[:5]

"""
Explanation: Converting the rawtext into sentences
End of explanation
"""


# using python's regular expressions to extract the required address data
street_address   = re.compile(u'\d{1,4} [\w\s]{1,20}(?:street|st|avenue|ave|road|rd|highway|hwy|square|sq|trail|trl|drive|dr|court|ct|park|parkway|pkwy|circle|cir|boulevard|blvd)\W?(?=\D|$)', re.IGNORECASE)
zip_code = re.compile(r'\b\d{5}(?:[-\s]\d{4})?\b')
proper_address = "[0-9]{1,5} .+, .+, [A-Z]{2} [0-9]{5}"


print ("Below are the street addresses found in the given document \n")
print (re.findall(street_address, raw_data))

print ("\nBelow are the zip codes found in the given document \n")
print (re.findall(zip_code, raw_data))

print ("\nBelow is the address found in the given document with PROPER format \n")
print (re.findall(proper_address, raw_data))
for i in range (len(sentences)):
    if sentences[i][0] == 'Address:':
        print ("\nBelow is the address found in the given document with IMPROPER format is: \n")
        print (sentences[i+1][0])


"""
Explanation: Task 1: To extract the addresses which are in different format in the given text
End of explanation
"""


tokens = [nltk.word_tokenize(sent[0]) for sent in sentences] 
tokens[:6]         # View the first few tokens

tokens[0]

"""
Explanation: Task 2: Entity Extraction
Tokenizing Words
End of explanation
"""


# Viewing the parts of speech tag for each token in the given text
pos_tags =  [nltk.pos_tag(token) for token_list in tokens for token in token_list] 
pos[:20]        # View top few pos tags

"""
Explanation: Entity Extraction
Parts of Speech Tagging
End of explanation
"""


nltk.help.upenn_tagset()

"""
Explanation: NNP - Noun Proper Plural tag with respec to NLP library. The below command provides description of each POS tag followed by the nltk library
End of explanation
"""


chunks = ne_chunk(pos)
print (chunks.__repr__())

"""
Explanation: Name Entity Chunks
The below command assigns name entity to each token in the given document. The commonly used type of named entities are
ORGANIZATION, PERSON, LOCATION, DATE, TIME, MONEY, PERCENT, FACILITY, GPE
End of explanation
"""


