# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # NLPeasy Workshop - Introduction
# # SDS 2020, 25th of June 2020

# In this demo you will use NLPEasy to quickly setup a Pandas-based pipeline. The results will then be saved in Elasticsearch and a Kibana dashboard is generated automatically to explore the texts and results.

# For the workshop you have 2 possibilities to participate:
# - **mybinder.org:** No setup needed, only a webbrowser, however only 1-2 GB RAM are available
# - **own laptop:** You need to install the python environment and either Docker or Elasticsearch+Kibana

# This is why we have to variants of the executen, a small one for binder and a bigger one to use on your laptop:

import os
DOIT_SMALL = 'BINDER_PORT' in os.environ

# You can overwrite the BINDER-setting if you need to or want to trye

# We are on bigger machines for the workshop:
DOIT_SMALL = False
"DOIT_SMALL: {}".format(DOIT_SMALL)

if DOIT_SMALL:
    nrows_to_load, spacy_model, spacy_cols = 1000, 'en_core_web_sm', ['myself_summary']
else:
    nrows_to_load, spacy_model, spacy_cols = 10000, 'en_core_web_md', None
"nrows_to_load: {} spacy_model: {} spacy_cols: {}".format(nrows_to_load, spacy_model, spacy_cols)

# ## Imports

# We will analyse today a dataset using NLPeasy package. We will also need pandas and numpy.

import pandas as pd
import numpy as np
import nlpeasy as ne
import util

# `util` is part of the workshop repository and helps with accessing the Kibana dashboard on binder.

# ## Connect to Elastic Stack

# Connect to running elastic instance on your local machine. If this can't be found, it will start an Open Source stack on your docker.
#
# By specifying mount_volume_prefix your Elastic data is saved.

elk = ne.connect_elastic(elk_version='7.10.2',
                         mount_volume_prefix='./elastic-data-live/')

# > If it is started on docker it will on the first time pull the images (1.3GB)!
# BTW, this function is not blocking, i.e. the servers might only be active couple of seconds later.
# Setting mountVolumePrefix="./elastic-data/" would keep the data of elastic in your
# filesystems and then the data survives container restarts

# On binder you do not have access to it's localhost. However set up some binder-jupyter magic so you can access it:

if util.BINDER:
   util.display_kibana_link()

# ## Read and Process Data

# We use 'OK Cupid' dataset in this workshop (the anonymized legal version of it).
# Let's load the data and see what we have here.

url = "profiles.csv.zip" if "profiles.csv.zip" in os.listdir() else "https://github.com/rudeboybert/JSE_OkCupid/raw/master/profiles.csv.zip"
print(f"Getting data from {url}")
okc = pd.read_csv('https://github.com/rudeboybert/JSE_OkCupid/raw/master/profiles.csv.zip', nrows=nrows_to_load)
okc

# We have some categorical / continuous variables about the OK cupid customers:
# * body type
# * diet
# * education
# * relgion
# * sex
# * astro sign
# * preferences for alcohol, cigarettes, drugs
# * sexual orientation
# * relationship status
# * language
# * location
#
#
# But they also needed to answer some questions (textual data):
#
# * essay0- My self summary
# * essay1- What I’m doing with my life
# * essay2- I’m really good at
# * essay3- The first thing people usually notice about me
# * essay4- Favorite books, movies, show, music, and food
# * essay5- The six things I could never do without
# * essay6- I spend a lot of time thinking about
# * essay7- On a typical Friday night I am
# * essay8- The most private thing I am willing to admit
# * essay9- You should message me if...
#
# Source: https://github.com/rudeboybert/JSE_OkCupid/blob/master/okcupid_codebook.txt  
#

# ### Drop some columns

okc.columns

okc = okc.drop(['essay1', 'essay2', 'essay3', 'essay4', 'essay5', 'essay6', 'essay8'], axis=1)

okc = okc.rename(columns={"essay0": "myself_summary", "essay7": "typical_friday_night", "essay9": "message_me_if"})

okc.head()

# ## NLPeasy Pipeline

# First step is to construct pipeline object:

pipeline = ne.Pipeline(index='okc-intro',
                       text_cols=['myself_summary', 'typical_friday_night', 'message_me_if'],
                       num_cols = ['age', 'height', 'income'],
                       tag_cols=['body_type', 'diet', 'drinks', 'drugs', 'education'],
                       elk=elk)

# Now let's add stages in the pipeline!

# NLPeasy pipelines can have as few or as many stages as you wish. Here we just use the following:
# * `VaderSentiment` is a nice rule-based sentiment prediction for english.
# * `Spacy` uses the `spacy` package together with it's language model - here english small (=sm) or medium (=md).
#     We use it for a couple of enrichments, as you will see below.
#
# There are more stages in NLPeasy you can use (e.g. RegexTag Extraction, Splitting) or you can define your own functions there.

# First the pipeline should calculate sentiments to all of the text columns. The first parameter specifies for which column, the second one stipulates the name of the output column:

pipeline += ne.VaderSentiment('myself_summary', 'myself_summary_sentiment')

# Now we will add NLPeasy's spaCy enrichemnt.

# For small environments (like on binder) do it just for one column. Also we then only use the small model which does not provide vectors.

f"spacy_model={spacy_model}"

# The spacy model takes ~30 secs to load:

pipeline += ne.SpacyEnrichment(nlp=spacy_model,
                               cols=['myself_summary'])

# ###### Run the pipeline - Spacy needs some time to process, so these ~60000 messages take about 100 minutes to process.

# We do this for a subset of the data as it will take too long otherwise.

okc_enriched = pipeline.process(okc.head(100), write_elastic=True, batchsize=20)

# Create Kibana Dashboard of all the columns

pipeline.create_kibana_dashboard()

# Open Kibana in webbrowser

util.display_kibana_link() if util.BINDER else elk.kibana

# Oh, and you wonder what took so long in the upload?

pipeline._tictoc.summary()



StarcoderdataJupyterScriptsDedupFiltered