#!/usr/bin/env python
# -*- coding: utf-8 -*-
from elasticsearch import Elasticsearch, helpers
from collections import Counter
from datetime import datetime


def find_pairs(list1, list2):
    pairs = []
    if list1 and list2:
        for item1 in list1:
            for item2 in list2:
                pairs.append(u'{}@{}'.format(item1, item2))
    return pairs


es = Elasticsearch()
index_name = 'embem'
doc_type = 'event'
cat1 = 'Body'
cat2 = 'Posemo'

timestamp = datetime.now().isoformat()

pairs_count = Counter()
years = {}

q = {
    "query": {
        "wildcard": {"text_id": "*"}
    }
}

results = helpers.scan(client=es, query=q, index=index_name, doc_type=doc_type)
for r in results:
    # get tags
    cat1_tags = r.get('_source').get('liwc-entities').get('data').get(cat1)
    cat2_tags = r.get('_source').get('liwc-entities').get('data').get(cat2)

    # find all pairs
    pairs = find_pairs(cat1_tags, cat2_tags)

    if pairs:
        for pair in pairs:
            pairs_count[pair] += 1
            year = r.get('_source').get('year')
            if year not in years.keys():
                years[year] = Counter()
            years[year][pair] += 1

    # save pairs to ES
    doc = {
        'doc': {
            'pairs-{}-{}'.format(cat1, cat2): {
                'data': pairs,
                'num_pairs': len(pairs),
                'timestamp': timestamp
            }
        }
    }
    es.update(index=index_name, doc_type=doc_type,
              id=r.get('_id'), body=doc)

sorted_years = years.keys()
sorted_years.sort()

print '{}\t{}\tFrequency'.format(cat1, cat2) + \
      ''.join(['\t{}'.format(k) for k in sorted_years])
print 'TOTAL\tTOTAL\t{}'.format(sum(pairs_count.values())) + \
      ''.join(['\t{}'.format(sum(years[k].values())) for k in sorted_years])
for p, f in pairs_count.most_common():
    (w1, w2) = p.split('@')
    print u'{}\t{}\t{}'.format(w1, w2, f).encode('utf-8') + \
          ''.join(['\t{}'.format(years[k][p]) for k in sorted_years])

Add script to generate pairs of LIWC categories
Add script to generate pairs of LIWC categories

Added a script that generates pairs of LIWC categories for Body and
Posemo words. The pairs are added to the index. And data about the pairs
is written to std out.
The LIWC categories for which pairs are generated are hardcoded. This
should be changed. Also adding the pairs to the ES index should be made
optional.
