# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Prequisites 
#  create and Elastic search domain using Elastic search services in AWS
#  
#  I changed follwoing settings and kept rest default (as it is). 
#  * From *Choose deployment type* chose *Development and testing*
#  * Can give any name to domain as long it satisfies the constrains given on that page
#  * From *Network configuration* chose *Public*
#  * From *Fine–grained access control – powered by Open Distro for Elasticsearch* chose *Create master user* and give any Master name and password. 
#  * From *Access policy* select *Allow open access to the domain*
#  
#  In _settings.py file copy the Master username and password

# make sure following are installed before importing, if not install using pip
import requests
from elasticsearch import Elasticsearch, RequestsHttpConnection
from requests_aws4auth import AWS4Auth
import boto3
from _settings import *
import json

host = 'search-irspring2021-5dfrmee6j25yj4rv65kudho3a4.us-west-2.es.amazonaws.com'
# host is url given as Endpoint of Domain you created, you do not need to write https
region = 'us-west-2' 
# can be seen from top right of aws console or also given in end point url for example us-west-2 in this case

# only used on AWS access keys are used
#service = 'es'
#credentials = boto3.Session().get_credentials()
#awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, service, session_token=credentials.token)


es = Elasticsearch(
    hosts = [{'host': host, 'port': 443}],
    http_auth = (MASTER, PASSWORD), # change to awsauth if authenticating in that way
    use_ssl = True,
    verify_certs = True,
    connection_class = RequestsHttpConnection
)

# +
# data
employee1 = {
    'first_name': 'john',
    'last_name':'smith',
    'age': 25,
    'about': 'I love rock climbing',
    'interests': ['sports', 'music']       
}
employee2 = {
    'first_name': 'jane',
    'last_name':'smith',
    'age': 32,
    'about': 'I like to collect rock albums',
    'interests': ['music']       
}

employee3 = {
    'first_name': 'jenny',
    'last_name':'smith',
    'age': 20,
    'about': 'I love rock climbing and playing football',
    'interests': ['sports', 'music']       
}
# -

# delete index if already exist 
es.indices.delete(index='megacorp', ignore=[400, 404])

# index the data
es.index(index="megacorp", doc_type="employee", id="1", body=employee1)
es.index(index="megacorp", doc_type="employee", id="2", body=employee2)
es.index(index="megacorp", doc_type="employee", id="3", body=employee3)

# get all documents from index
results= es.search(index='megacorp')
print(json.dumps(results, indent=1))

# query to get document by id
results= es.get(index="megacorp", doc_type="employee", id="3")
print(json.dumps(results, indent=1))

# match 
results= es.search(index='megacorp', explain='true', body={'query': {'match': {'about':"rock climbing"}}})
print(json.dumps(results, indent=1))

# phrase match
results= es.search(index="megacorp", explain='true', body={"query": {"match_phrase": {'about':"rock climbing"}}})
print(json.dumps(results, indent=1))

# Notice that N and n  while finding idf are 1, this is because these values are caculated  from each shard, not from whole dataset.
#
# This will not be issue if data is large, idf accross shards ~= to IDF for all data
#
# However, if you want to use IDF for whole data add *search_type='dfs_query_then_fetch'* paramter to search request 

# match with search_type='dfs_query_then_fetch'
results= es.search(index='megacorp', explain='true', search_type='dfs_query_then_fetch', body={'query': {'match': {'about':"rock climbing"}}})
print(json.dumps(results, indent=1))

# match_phrase with search_type='dfs_query_then_fetch'
results= es.search(index='megacorp', explain='true', search_type='dfs_query_then_fetch', body={'query': {'match_phrase': {'about':"rock climbing"}}})
print(json.dumps(results, indent=1))





StarcoderdataJupyterScriptsDedupFiltered