import numpy as np
import pandas as pd
import lxml.etree as ET
import re
from sklearn.feature_extraction.text import CountVectorizer

"""
Explanation: Analyzing Patents With USPTO Open Data
Welcome! This Jupyter Notebook will go through the process of reading patent data, analyzing the word usage in the patent abstracts, and identifying words which are used more frequently by city or state. Hopefully, we can learn about regional differences in patent filing.
This analysis is possible because the USPTO makes bulk patent data available online through a collaboration with Reed Tech (http://patents.reedtech.com/). 
Questions, comments, suggestions, and corrections can be directed to mgebhard@gmail.com.
If you want to work through the code and follow along on your own computer, you'll need the python packages: lxml, pandas, sklearn, re, and numpy.
End of explanation
"""


files = ['ipg160119.xml', 'ipg150120.xml', 'ipg160816.xml', 'ipg160809.xml', 'ipg160802.xml',
         'ipg160719.xml', 'ipg160726.xml']

"""
Explanation: Download and Preprocess the Data

Download .ZIP files from http://patents.reedtech.com/pgrbft.php. Because of the different formatting, we'll stick to files that fall into 2001-Present. Each file contains all of the patents issued by the USPTO for one week. These files are each around 100 MB zipped and unzip to 300-700 MB so we can't work with too many at once. Extract them to the working directory.
The more patent data we can analyze at once, the more our word frequency output will be due to true regional differences and not just low sample sizes. Let's make a list of file paths from some recent releases so we can process multiple files.
End of explanation
"""


class patent(object):
    def __init__(self, location, abstract):
        self.location = location
        self.abstract = abstract
    def getLocation(self):
        return self.location
    def getAbstract(self):
        return self.abstract
    
def filesToDataFrame(files):
    patents = []
    for path in files:
        with open(path, 'rb') as f:
            lines = f.readlines()
        patent_entries_start = []
        for line in range(len(lines)):
            if lines[line][0:5] == '<?xml':
                patent_entries_start.append(line)
        for i in range(len(patent_entries_start)-1):
            start = patent_entries_start[i]
            stop = patent_entries_start[i+1]
            patent_entry = lines[start:stop]
            location, abstract = process_entry(patent_entry)
            patents.append(patent(location, abstract))
    patent_df = pd.DataFrame.from_records([(x.getLocation()[0], 
                                            x.getLocation()[1], 
                                            x.getLocation()[2], 
                                            x.getAbstract()) for x in patents], 
                                              columns=['city', 'state', 'country', 'abstract'])
    patent_df.dropna(inplace=True)
    return patent_df

def process_entry(entry):
    with open('temp.xml', 'w') as xml_entry:
        for line in entry:
            xml_entry.write(line)
    context = ET.iterparse('temp.xml', events=('start', 'end'))
    city = None
    state = None
    country = None
    paragraph = None
    for event, elem in iter(context):
        if event == 'start' and elem.tag == 'inventor':
            for child in elem:
                for grandchild in child:
                    for info in grandchild:
                        if info.tag == 'city':
                            city = info.text
                        if info.tag == 'state':
                            state = info.text
                        if info.tag == 'country':
                            country = info.text
        if event == 'start' and elem.tag == 'abstract':
            paragraph = elem.findtext('p')
    return (city, state, country), paragraph

patent_df = filesToDataFrame(files)

"""
Explanation: We want to extract location data and the abstract text from each patent in these files. Unfortunately, the files are not properly constructed .XML files but instead contain a concatenation of individual .XML files for each patent, which makes reading them more difficult. We have to use a bit of code to read these files and extract the data to a pandas dataframe.
End of explanation
"""


patent_df.info()
patent_df.sample(10)

"""
Explanation: Let's take a look at the dataframe we've constructed.
End of explanation
"""


def getOverallWordFrequency(patent_df, vocab_size):
    combinedWords = ''
    for entry in patent_df['abstract']:
        combinedWords += entry + ' '
    text = processText(combinedWords)
    cv = CountVectorizer(min_df=0, 
                         decode_error='ignore', 
                         stop_words='english', 
                         max_features=vocab_size)
    counts = cv.fit_transform([text]).toarray().ravel()
    words = np.array(cv.get_feature_names())
    counts = counts / float(counts.max())
    return zip(words, counts)

def processText(text):
    text = text.lower()
    text = re.sub(r'\W', ' ', text)
    text = text.split()
    return ' '.join(text)

"""
Explanation: Find Word Frequencies in a String of Text
We now want to write a function that can take in our dataframe (or a piece of it) and keep track of word usage in the abstracts that appear in the dataframe. To do this, we take the following steps:
1. Combine all of the abstracts into one long string.
2. Process the string to remove uppercase letters and non-alphanumeric characters.
3. Use CountVectorizer from sklearn.feature_extraction.text to convert the text to a matrix of token counts. This is especially useful because it removes "stop words" which are short, common words that don't have meaning for our purpose. We can also limit the vocab size using max_features. In other words, we can choose to only look at, for example, the top 200 most frequent words.
    Documentation here: http://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html
End of explanation
"""


def getHighestScoredWordInRegion(regionalUsage, overallUsage):
    highestScore = 0
    wordWithHighestScore = None
    for wordAndFreq in regionalUsage:
        overallFreq = getOverallFreq(wordAndFreq[0], overallUsage)
        score = getScore(wordAndFreq[1], overallFreq)
        if score > highestScore:
            wordWithHighestScore = wordAndFreq[0]
            highestScore = score
    return wordWithHighestScore

def getOverallFreq(word, overallUsage):
    for wordAndFreq in overallUsage:
        if word == wordAndFreq[0]:
            return wordAndFreq[1]
    return 1

"""
Explanation: Compare Regional Word Frequencies to Overall Word Frequencies
Now that we can generate a list of words and their occurrances in the patent abstracts, we need to be able to compare a list created from a region with the list created from the overall data.
End of explanation
"""


def getScore(singleFreq, overallFreq):
    return (singleFreq / overallFreq)

"""
Explanation: Above, we used the getScore function to assign a score to a word based on its frequency in the region compared to its frequency in the overall data. A simple scoring function is just the ratio of the occurrance in the region to the occurrance in the overall data. Other scoring functions will, of course, identify different patterns in word usage.
End of explanation
"""


def bestWordInRegion(region, city=False, state=False):
    overallUsage = getOverallWordFrequency(patent_df, 100000)
    output = []
    for location in region:
        if city:
            regionalUsage = getOverallWordFrequency(patent_df[patent_df['city'] == location], 500)
            output.append((location, 
                           len(patent_df[patent_df['city'] == location].index), 
                           getHighestScoredWordInRegion(regionalUsage, overallUsage)))
        if state:
            regionalUsage = getOverallWordFrequency(patent_df[patent_df['state'] == location], 200)
            output.append((location, 
                           len(patent_df[patent_df['state'] == location].index), 
                           getHighestScoredWordInRegion(regionalUsage, overallUsage))) 
    output_df = pd.DataFrame.from_records(output, columns=['location', 'patents analyzed', 'frequent word'])
    print output_df
    return

"""
Explanation: We can put this all together into a final function:
End of explanation
"""


region_states = ['AL', 'AR', 'CA', 'CO', 'CT', 'DE', 'FL', 'GA', 'HI', 'ID', 'IL', 'IN', 'IA', 'KS', 
                 'KY', 'LA', 'ME', 'MD', 'MI', 'MN', 'MS', 'MO', 'MT', 'NE', 'NV', 'NH', 'NJ', 'NM', 'NY',
                 'NC', 'ND', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC', 'SD', 'TN', 'TX', 'UT', 'VT', 'VA', 'WA',
                 'WV', 'WI', 'WY']
bestWordInRegion(region_states, state=True)

"""
Explanation: Look at Word Usage by State and City
The function bestWordInRegion takes in a list of city or state regions and tells us the word for each region that stand out. Let's look at states. Alaska is left off since it has so few patents.
End of explanation
"""


region_cities = ['San Jose', 'Santa Clara', 'Sunnyvale', 'Cupertino', 'Mountain View', 'Palo Alto',
                 'Los Altos', 'Menlo Park', 'Redwood City', 'San Mateo', 'South San Francisco',
                 'San Francisco', 'Berkeley', 'Oakland']
bestWordInRegion(region_cities, city=True)

"""
Explanation: Additionally, we can look at patents by city. Granted, this may not be as accurate as analyzing by state for a few reasons. (1) There are likely fewer patents from a city than from a state. (2) The location is based on only one inventor when there are often multiple inventors listed per patent. (3) The location is based on the inventor's residence address and not necessarily the city where the work was done.
But analyzing by city can still be interesting. Let's look at some cities in the Bay Area.
End of explanation
"""


