# coding: utf-8

import os

from cheshire3.baseObjects import Session
from cheshire3.document import StringDocument
from cheshire3.internal import cheshire3Root
from cheshire3.server import SimpleServer   

session = Session()
session.database = 'db_dickens'
serv = SimpleServer(session, os.path.join(cheshire3Root, 'configs', 'serverConfig.xml'))
db = serv.get_object(session, session.database)
qf = db.get_object(session, 'defaultQueryFactory')
resultSetStore = db.get_object(session, 'resultSetStore')
idxStore = db.get_object(session, 'indexStore')

"""
Explanation: Basic Cheshire objects and methods
This file wants to document how one can use Cheshire to query the CLiC database. 
It fills in the gaps of the official cheshire documentation and it 
provides a number of very specific, hands-on examples.
Author: Johan de Joode
Dates: 10/2/2015
Database: Dickens
Setup
End of explanation
"""


query = qf.get_query(session, 'c3.subcorpus-idx = "dickens" and/cql.proxinfo c3.chapter-idx = "fog"')

"""
Explanation: Querying
Build a query. This does not hit the database itself.
End of explanation
"""


print query.toCQL()

print query.toXCQL()

"""
Explanation: A query can be printed as CQL or as XCQL
End of explanation
"""


result_set = db.search(session, query)
result_set

"""
Explanation: To search the database using this particular query, one needs to use the search method on a database object. This spits out a result set.
End of explanation
"""


print len(result_set)

"""
Explanation: Handling the results
When using the chapter index, the result set is an iterable of results in which each chapter is one result. For the query above, there are thus 35 chapters that match the query.
End of explanation
"""


for result in result_set:
    print result

"""
Explanation: Each result in the result set refers to a particular recordStore in which, surprise surprise, the actual chapter is stored.
End of explanation
"""


for result in result_set:
    print 'result.id: ', result.id
    print 'result.database: ', result.database
    print 'result.occurrences: ', result.occurences
    print 'result.proxInfo: ', result.proxInfo
    print "#########"

for result in result_set:
    print result.attributesToSerialize

"""
Explanation: Understanding the results
For each of these results a number of attributes can be accessed using the dot notation. The choices are:

result.attributesToSerialize  
result.id                     
result.recordStore
result.database
result.diagnostic
result.fetch_record
result.proxInfo
result.weight
result.numericId
result.resultSet
result.occurences
result.serialize
result.scaledWeight 

In our current setup it seems that results are not weighed. 
proxInfo is one of the most important attributes for our purposes.
It describes the proximity information for a hit in a particular record,
or in other words, where in a record the search string can be found.
We currently assume the following values:
* the first item is the id of the root element from
which to start counting to find the word node
for instance, 0 for a chapter view (because the chapter
is the root element), but 151 for a search in quotes
text.
* the second item in the deepest list (169, 171)
is the id of the <w> (word) node
* the third element is the character offset, 
or the exact character (spaces, and
and punctuation (stored in <n> (non-word) nodes
at which the search term starts
* the fourth element is the total amount of characters
 in the document
End of explanation
"""


for result in result_set:
    rec = result.fetch_record(session)
    print type(rec), rec

"""
Explanation: From what I gather, a result in a resultSet is only a pointer to the document and not the document itself.
The latter needs to be fetched and is generally called a record.
Records have the following attributes (most of which seem irrelevant for our purposes and several of which
only return empty strings):
rec.baseUri           rec.elementHash       rec.get_sax           rec.parent            rec.rights            
rec.byteCount         rec.fetch_proxVector  rec.get_xml           rec.processHistory    rec.sax               
rec.context           rec.fetch_vector      rec.history           rec.process_xpath     rec.size              
rec.digest            rec.filename          rec.id                rec.recordStore       rec.status            
rec.dom               rec.get_dom           rec.metadata          rec.resultSetItem     rec.tagName  
rec.wordCount         rec.xml
End of explanation
"""


for result in result_set:
    rec = result.fetch_record(session)
    print "rec.id: ", rec.id
    print 'rec.wordCount: ', rec.wordCount
    print 'rec.get_dom(session): ', rec.get_dom(session)
    print "#######"

result_set.attributesToSerialize

result.attributesToSerialize

for result in result_set:
    print result.serialize(session)

"""
Explanation: The get_dom(session) method spits out the record in parsed xml.
This is essential for our purposes.
End of explanation
"""


from lxml import etree
rec_tostring = etree.tostring(rec2)
print rec_tostring

"""
Explanation: A record can be transformed into raw xml (in order to understand it), using
a method from lxml:
End of explanation
"""


# find the first occurrence of the term love
# because that is what we are all looking for
love = rec_tostring.find('love')
conc_line = rec_tostring[love-50 : love + len('love') + 50]
conc_line.replace('love', 'LOVE')

"""
Explanation: This could also be used in simple python string manipulations. 
For instance, to highlight something in a chapter, or to build 
a concordance based on the raw string rather than an xml tree.
In that case one should note that only each occurrence of a term is 
duplicated because it is present in &lt;txt&gt; and in its own word node.
End of explanation
"""


kwicTransformer = db.get_object(session, 'kwic-Txr')

print kwicTransformer

doc = kwicTransformer.process_record(session, rec).get_raw(session)

print doc

from cheshire3.transformer import XmlTransformer

dctxr = db.get_object(session, 'kwic-Txr')

dctxr

doc = dctxr.process_record(session, record)
print doc.get_raw(session)[:1000]

"""
Explanation: Transforming a result
Rather than manually handling the xml like this, Cheshire has a class called a
Transformer that can perform xsl transformations on the xml of a chapter.
Transformers are defined in a configuration file. In our project they live in an
xsl file.
The following examples use a transformer that was not designed to work with our input, 
but they do illustrate how transformers can be invoked.
End of explanation
"""


query = qf.get_query(session, 'c3.book-idx = "LD"')
result_set = db.search(session, query)

chapter_1 = result_set[0]
chapter_44 = result_set[43]

chapter_1

rec = chapter_1.fetch_record(session).get_dom(session)
print rec

rec.attrib

rec.attrib['id']

type(rec)

print rec

doc = kwicTransformer.process_record(session, chapter_1.fetch_record(session)).get_raw(session)

print doc

articleTransformer = db.get_object(session, 'article-Txr')

doc = articleTransformer.process_record(session, chapter_1.fetch_record(session)).get_raw(session)

print doc

#FIXME How can you get immediately query for a chapter, 
# rather than getting all chapters of a book first?
# --> you need to build a better index for this
query = qf.get_query(session, 'c3.book-idx "LD" and div.id = "LD.1"')

result_set = db.search(session, query)

len(result_set)

#TODO if recordStore's are unique AND they represent chapters, it could also be possible to simply 
# get a particular recordStore from Cheshire (without querying the database again).


"""
Explanation: Retrieving a chapter
End of explanation
"""


query = qf.get_query(session, 'c3.subcorpus-idx = "dickens" and c3.chapter-idx = "fog" and c3.book-idx = "BH"')
result_set = db.search(session, query)
len(result_set)

"""
Explanation: Searching in a specific book
End of explanation
"""


query = qf.get_query(session, 'c3.subcorpus-idx = "dickens" \
                            and/cql.proxinfo c3.chapter-idx = "dense fog" \
                            ') #and c3.chapter-idx = "dense"')

rs = db.search(session, query)

len(rs)

for result in rs:
    print result.proxInfo
    #FIXME it seems that occurences cannot be trusted?
    print result.occurences

query = qf.get_query(session, 'c3.subcorpus-idx = "dickens" \
                            and/cql.proxinfo c3.chapter-idx = "the" \
                            ')

query.addPrefix(query, 'test')

query.toCQL()

"""
Explanation: Messing around
End of explanation
"""


query = qf.get_query(session, 'c3.subcorpus-idx = "dickens" \
                            and/proxinfo c3.chapter-idx = "dense fog" \
                            ')

rs = db.search(session, query)

total = 0
for result in rs:
    total += len(result.proxInfo)

total

"""
Explanation: Phrase search
End of explanation
"""


query = qf.get_query(session, 'c3.subcorpus-idx = "dickens" \
                            and/cql.proxinfo c3.chapter-idx = "fog" \
                            and c3.chapter-idx = "dense"')

rs = db.search(session, query)

len(rs)

"""
Explanation: And search
End of explanation
"""


query = qf.get_query(session, 'c3.subcorpus-idx = "dickens" \
                            and/cql.proxinfo c3.chapter-idx = "fog" \
                            or c3.chapter-idx = "dense"')
rs = db.search(session, query)
len(rs)

query = qf.get_query(session, 'c3.subcorpus-idx = "dickens" \
                               and c3.book-idx = "LD"')
rs = db.search(session, query)
len(rs)

query = qf.get_query(session, 'c3.subcorpus-idx = "dickens" \
                               and c3.chapter-idx = "he" prox/distance=1/unordered c3.chapter-idx = "said" \
                               or c3.chapter-idx = "did" or c3.chapter-idx = "wanted"')
rs = db.search(session, query)
len(rs)

#TODO not
#TODO wildcards

query = qf.get_query(session, 'c3.subcorpus-idx = "dickens" \
                               and c3.chapter-idx window/distance<5/unordered "low voice"')
rs = db.search(session, query)
len(rs)

for result in rs:
    print result.proxInfo

query = qf.get_query(session, 'c3.subcorpus-idx = "dickens" \
                               and c3.chapter-idx window/distance<5/unordered "voice low"')
rs = db.search(session, query)
len(rs)

query = qf.get_query(session, 'c3.subcorpus-idx = "dickens" \
                               and c3.chapter-idx window/distance<5/unordered "low high"')
rs = db.search(session, query)
len(rs)

query = qf.get_query(session, 'c3.subcorpus-idx = "dickens" \
                               and c3.chapter-idx window/distance<3 "Mr Arthur said"')
rs = db.search(session, query)
len(rs)

"""
Explanation: Or search
End of explanation
"""


query = qf.get_query(session, '(c3.subcorpus-idx all "dickens" and/cql.proxinfo c3.chapter-idx any "dense fog")')
result_set = db.search(session, query)
count = 0
for result in result_set:
    record = result.fetch_record(session)
    print result.occurences, record #wordCount #.process_xpath('//w[@o=%s]' % result.proxInfo[0][1])
    for y in result.proxInfo:
        print y
        count += 1

#TODO why does proxinfo only have three values here?
# --> because the last any does not have a proxinfo value

"""
Explanation: Proximity Information
End of explanation
"""


from cheshire3.transformer import LxmlQueryTermHighlightingTransformer

"""
Explanation: Term highlighting
End of explanation
"""


