from partitioner import partitioner
from partitioner.methods import *

"""
Explanation: Partitioner examples
This is a jupyter notebook with a few vignettes that present some of the Python partitioner package's functionality.
Note: Cleaning of text and determination of clauses  occurs in the partitionText method.  Because of this, it is unwise to pass large, uncleaned pieces of text as 'clauses' directly through the .partition() method (regardless of the type of partition being taken), as this will simply tokenize the text by splitting on " ", producing many long, punctuation-filled phrases, and likely run very slow. As such, best practices only use .partition() for testing and exploring the tool on case-interested clauses.
End of explanation
"""


## Vignette 1: Build informed partition data from a dictionary, 
##             and store to local collection
def preprocessENwiktionary():
    pa = partitioner(informed = True, dictionary = "./dictionaries/enwiktionary.txt")
    pa.dumpqs(qsname="enwiktionary")

preprocessENwiktionary()

"""
Explanation: Process the English Wiktionary to generate the (default) partition probabilities.
Note: this step can take significant time for large dictionaries (~5 min).
End of explanation
"""


## Vignette 2: An informed, one-off partition of a single clause
def informedOneOffPartition(clause = "How are you doing today?"):
    pa = oneoff()
    print pa.partition(clause)

informedOneOffPartition()
informedOneOffPartition("Fine, thanks a bunch for asking!")

"""
Explanation: Perform a few one-off partitions.
End of explanation
"""


## Vignette 3: An informed, stochastic expectation partition of a single clause
def informedStochasticPartition(clause = "How are you doing today?"):
    pa = stochastic()
    print pa.partition(clause)

informedStochasticPartition()

"""
Explanation: Solve for the informed stochastic expectation partition (given the informed partition probabilities).
End of explanation
"""


## Vignette 4: An uniform, one-off partition of a single clause
def uniformOneOffPartition(informed = False, clause = "How are you doing today?", qunif = 0.25):
    pa = oneoff(informed = informed, qunif = qunif)
    print pa.partition(clause)

uniformOneOffPartition()
uniformOneOffPartition(qunif = 0.75)

"""
Explanation: Perform a pure random (uniform) one-off partition.
End of explanation
"""


## Vignette 5: An uniform, stochastic expectation partition of a single clause
def uniformStochasticPartition(informed = False, clause = "How are you doing today?", qunif = 0.25):
    pa = stochastic(informed = informed, qunif = qunif)
    print pa.partition(clause)

uniformStochasticPartition()
uniformStochasticPartition(clause = "Fine, thanks a bunch for asking!")

"""
Explanation: Solve for the uniform stochastic expectation partition (given the uniform partition probabilities).
End of explanation
"""


## Vignette 6: Use the default partitioning method to partition the main partitioner.py file and compute rsq
def testPartitionTextAndFit():
    pa = oneoff()
    pa.partitionText(textfile = pa.home+"/../README.md")
    pa.testFit()
    print "R-squared: ",round(pa.rsq,2)
    print
    phrases = sorted(pa.counts, key = lambda x: pa.counts[x], reverse = True)
    for j in range(25):
        phrase = phrases[j]
        print phrase, pa.counts[phrase]

testPartitionTextAndFit()

"""
Explanation: Build a rank-frequency distribution for a text and determine its Zipf/Simon (bag-of-phrase) $R^2$.
End of explanation
"""


## Vignette X1: Build informed partition data from other dictionaries, 
##             and store to local collection
def preprocessOtherWiktionaries():
    for lang in ["ru", "pt", "pl", "nl", "it", "fr", "fi", "es", "el", "de", "en"]:
        print "working on "+lang+"..."
        pa = partitioner(informed = True, dictionary = "./dictionaries/"+lang+".txt")
        pa.dumpqs(qsname=lang)

preprocessOtherWiktionaries()

"""
Explanation: Process the some other Wiktionaries to generate the partition probabilities.
Note: These dictionaries are not as well curated and potentially contain phrases from other languages (a consequence of wiktionary construction). As a result, they hold many many more phrases and will take longer to process. However, since the vast majority of these dictionaries are language-correct, effects on the partitioner and its (course) partition probabilities is likely negligable.
End of explanation
"""


from partitioner import partitioner
from partitioner.methods import *
## Vignette X2: Use the default partitioning method to partition the main partitioner.py file and compute rsq
def testFrPartitionTextAndFit():
    for lang in ["ru", "pt", "pl", "nl", "it", "fr", "fi", "es", "el", "de", "en"]:
        pa = oneoff(qsname = lang)
        pa.partitionText(textfile = "./tests/test_"+lang+".txt")
        pa.testFit()
        print
        print lang+" R-squared: ",round(pa.rsq,2)
        print
        phrases = sorted(pa.counts, key = lambda x: pa.counts[x], reverse = True)
        for j in range(5):
            phrase = phrases[j]
            print phrase, pa.counts[phrase]

testFrPartitionTextAndFit()

"""
Explanation: Test partitioner on some other languages.
End of explanation
"""


