# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ## JSON

# # JSON - Javascript Object Notation
# #### Invented by <NAME> when working at Yahoo in early 2000s.
#
# * Goal - Human Readable, Machine Parsable
#
# * Specification: https://www.json.org/

# JSON — short for JavaScript Object Notation — format for sharing data. 
#
# JSON is derived from the JavaScript programming language
#
# Available for use by many languages including Python 
#
# usually file extension is .json when stored
#
#

# +
# Sample JSON below from https://json.org/example.html
# Question why is Syntax highlighting working properly ? :)
# -

{"widget": {
    "debug": "on",
    "window": {
        "title": "Sample Konfabulator Widget",
        "name": "main_window",
        "width": 500,
        "height": 500
    },
    "image": { 
        "src": "Images/Sun.png",
        "name": "sun1",
        "hOffset": 250,
        "vOffset": 250,
        "alignment": "center"
    },
    "text": {
        "data": "Click Here",
        "size": 36,
        "style": "bold",
        "name": "text1",
        "hOffset": 250,
        "vOffset": 100,
        "alignment": "center",
        "onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;"
    }
}}    


# if this was string starting with { it would be our json
mydata = {
    "firstName": "Jane",
    "lastName": "Doe",
    "hobbies": ["running", "sky diving", "dancing"],
    "age": 43,
    "children": [
        {
            "firstName": "Alice",
            "age": 7
        },
        {
            "firstName": "Bob",
            "age": 13
        }
    ]
}

type(mydata)

print(mydata)

mydata['children'][1]['age']

mydata['children'][-1]['age']

mydata.get('hobbies')

mydata.get('hobbies')[-1],mydata['hobbies'][-1]



mylist = list(range(10))
print(mylist)

# The process of encoding JSON is usually called serialization. This term refers to the transformation of data into a series of bytes (hence serial) to be stored or transmitted across a network. You may also hear the term marshaling, but that’s a whole other discussion. Naturally, deserialization is the reciprocal process of decoding data that has been stored or delivered in the JSON standard.
#
# All we’re talking about here is reading and writing. Think of it like this: encoding is for writing data to disk, while decoding is for reading data into memory.
#  https://realpython.com/python-json/

mydata # simply a PYthon dictionary with some lists inside etc

# we need a library for decoding and encoding json
import json

# first we are going to dump our data into a text file
with open("data_file.json", mode="w") as write_file:
    json.dump(mydata, write_file)
# remember that stream is closed here and file is written by now

# this will be nicer
with open("data_file_indented.json", mode="w") as write_file:
    json.dump(mydata, write_file, indent=4)

with open("data_file_indented.json") as f:
    raw_txt = f.read()
raw_txt[:150]

# deserialize, decode from json string into Python Data structure
my_data = json.loads(raw_txt)
type(my_data)

my_data.keys()

my_data['children']

# more often we will load json immediately
with open("data_file_indented.json") as f:
    my_data_2 = json.load(f)
type(my_data_2)

# contents are the same but two different objects
my_data == my_data_2, my_data is my_data_2

my_json_string = json.dumps(my_data) # converst Python data structure into json string
my_json_string[:100]

type(my_json_string)

json.loads(my_json_string)

json.dumps(my_data)

import requests # this library is not included with Python but is very popular and comes with Anaconda
# pip install requests otherwise

# we make a http request to a url and print the response code
url = "https://my.api.mockaroo.com/ageincluded.json?key=<KEY>"
response = requests.get(url)
print(response.status_code) # Response Code 200 is good!

data_from_json = response.json() # we do not need json.loads
type(data_from_json), len(data_from_json)

response.text[:50]

my_mock_data = json.loads(response.text)
type(my_mock_data), len(my_mock_data)

data_from_json == my_mock_data, data_from_json is my_mock_data

import pandas as pd

df = pd.read_json(url)
df.head()

df.to_csv("mock_data.csv")

# +
# idea get average age of Japanese Men in our JSON data


# -

data_from_json[:5]

myjson = data_from_json # just an alias

japanese = [person for person in myjson if person.get('email').endswith('.jp')]
# potentially person.get('email') could return None then .endswith('.jp') would fail with error
japanese

japanese_men = [p for p in japanese if p.get('gender') == "Male"]
# potentially person.get('email') could return None then .endswith('.jp') would fail with error
japanese_men

uk = [person for person in myjson if person.get('email',"").endswith('.uk')]
uk

uk_ages = [(person.get('first_name'),int(person.get('age'))) for person in uk]
uk_ages

jp_ages = [(person.get('first_name'),int(person.get('age'))) for person in japanese]
jp_ages

with open("uk_ages.json", mode="w") as fstream:
    json.dump(uk_ages, fstream, indent=4)

# so we lose the tuple designed when writing to JSON and back
with open("uk_ages.json") as fstream:
    uk_data = json.load(fstream)
uk_data

with open("jp_ages.json", mode="w") as fstream:
    json.dump(jp_ages, fstream, indent=4)



type(mydata)



type(json_string)

type(mydata)

# Convert Json_string back to our Python Object
my_obj = json.loads(json_string)
my_obj

my_obj.get('firstName')

mydata

newlist = json.loads('[1,3,5,"Valdis"]')
newlist

type(newlist)

badlist = json.loads('[1,3,5,"Vald]",334342]')
badlist



type(json_string)

# +
# Avove example JSON and Python object have the same syntax but there are some differences
# -

# ![object](../img/object.png)

# ![Array](../img/array.png)

# ![Value](../img/value.png)

# Simple Python objects are translated to JSON according to a fairly intuitive conversion.
#
# Python	JSON
#
# dict	object
#
# list, tuple	array
#
# str	string
#
# int, long, 
#
# float	number
#
# True	true
#
# False	false
#
# None	null

newlist = json.loads('[true,2,null, false, 555.333]')
newlist

# +
# The first option most people want to change is whitespace. You can use the indent keyword argument to specify the indentation size for nested structures. Check out the difference for yourself by using data, which we defined above, and running the following commands in a console:

json.dumps(mydata)

# -

# very useful for visibility!
print(json.dumps(mydata, indent=4))

with open("data_file.json", "w") as write_file:
    json.dump(mydata, write_file, indent=4)

# how to read json into our python data obj from file
with open("data_file.json", "r") as read_file:
    data = json.load(read_file)
data

type(data)

len(data)

list(data.items())

numberedlist = list(enumerate(data.items()))
numberedlist

# save numberedlist with ident = 4 in file numberedlist.json
with open('numberedlist.json', mode='w') as f:
    json.dump(numberedlist, f, indent=4)


# Keep in mind that the result of this method could return any of the allowed data types from the conversion table. This is only important if you’re loading in data you haven’t seen before. In most cases, the root object will be a dict or a list.

# If you've gotten JSON data in from another program or have otherwise obtained a string of JSON formatted data in Python, you can easily deserialize that with loads(), which naturally loads from a string:

json_string = """
{
    "researcher": {
        "name": "<NAME>",
        "species": "Betelgeusian",
        "relatives": [
            {
                "name": "<NAME>",
                "species": "Betelgeusian"
            }
        ]
    }
}
"""
data = json.loads(json_string)
data

# get value of relative's name
data['researcher']

# get value of relative's name
data['researcher']['relatives']

# get value of relative's name
data['researcher']['relatives'][0]

# get value of relative's name
data['researcher']['relatives'][0]['name']

data['researcher']['relatives'][0]['name'].split()[0]

data['researcher']['relatives'][0]['name'].split()[0][:4]

type(data)

import json
import requests

# +
## Lets get some data https://jsonplaceholder.typicode.com/
# -

response = requests.get("https://jsonplaceholder.typicode.com/todos")
if response.status_code != 200:
    print("Bad Response: ", response.status_code)
print(response.status_code)
todos = json.loads(response.text)


# can open https://jsonplaceholder.typicode.com/todos in regular browser too..

type(todos)

len(todos)

todos[:10]

# completedworks = [el for el in todos if el['completed'] == True]
completedworks = [el for el in todos if el.get('completed') == True]
len(completedworks)

completedworks[-10:]

type(completedworks)

users = {}
for el in completedworks:
    k = el['userId']
    if k in users:
        users[k] +=1
    else:
        users[k] = 1
users

sorted(users.items(),key = lambda el: el[1], reverse=True)

from collections import Counter

count = Counter([el['userId'] for el in completedworks])
count.most_common()

# lets do everything at once
finishedcount = Counter([el.get('userId') for el in todos if el.get('completed') == True])
finishedcount.most_common()

import matplotlib.pyplot as plt

plt.bar(finishedcount.keys(), finishedcount.values())
plt.show()

todos[-3:]

[1,2] + [3,4,6,6]

todos += [{'completed':True},{'completed':True},{'completed':True},{'completed':True}]
todos[-5:]

# lets do everything at once
finishedcount = Counter([el.get('userId') for el in todos if el.get('completed') == True])
finishedcount.most_common()

myl = [('Valdis', 40), ('Alice',35), ('Bob', 23),('Carol',70)]


# +
# Lambda = anonymous function
# -

def myfun(el):
    return el[1]
# same as myfun = lambda el: el[1]


sorted(myl, key = lambda el: el[1], reverse=True)

# +
# Exercise find out top 3 users with most tasks completed!

# TIPS
# we need some sort of structure to store these user results before finding out top 3
# at least two good data structure choices here :)
# here the simplest might actually be the best if we consider userId values

# -

todos[0]

todos[0]['userId']

todos[0]['completed']

# Here we create a new dictionary and and count the completed works by id
newdict = {}
for todo in todos:
    if todo['completed'] == True:
        if todo['userId'] in newdict:
            newdict[todo['userId']] += 1
        else:
            newdict[todo['userId']] = 1

newdict

sorted(newdict.items())

bestworkers = sorted(newdict.items(), key=lambda el: el[1], reverse=True)
bestworkers[:3]

users = [ el['userId'] for el in todos]
len(users),users[:15]

uniqusers = set(users)
uniqusers

# dictionary comprehension but could live without one
users = { el['userId'] : 0 for el in todos} 

users

users.keys()

users.value

# +
#{'completed': True,
# 'id': 8,
#  'title': 'quo adipisci enim quam ut ab',
#  'userId': 1}
# -



#idiomatic
for el in todos:
    users[el['userId']] += el['completed'] # Boolean False is 0 True is 1 obviously this might not be too readable

# same as above could be useful in more complicated cases
for el in todos:
    if el['completed'] == True:
        users[el['userId']] += 1

# +
# there could be a one liner or a solution with from collections import Counter
# -

users.items()

list(users.items())

userlist=list(users.items())

type(userlist[0])

# we pass a key anonymous(lambda) function
sorted(userlist, key=lambda el: el[1], reverse=True)[:3]

# +
# lets try a simple way
# -

mylist=[0]
mylist*=11

for el in todos:
    if el['completed'] == True:
        mylist[el['userId']] +=1

mylist

mylist.index(max(mylist))

# +
# kind of hard to get more values need to get tricky
# -

# # How about Pandas and Json ?

import pandas as pd

df = pd.read_json('https://jsonplaceholder.typicode.com/todos')

df.head()

df.to_csv('my_todos.csv')

df.shape

df.describe()

df.describe(include=['O'])

# we see that completed
df.groupby(['userId']).sum()

df.groupby(['userId']).sum()['completed'].plot(kind="bar")

busyjson

df.groupby(['userId'])['completed'].sum()

# if we need a single column dataframe
df.groupby(['userId'])[['completed']].sum()

df.groupby(['userId'])['completed'].sum().sort_values()

df.groupby(['userId'])['completed'].sum().sort_values(ascending=False)

busyjson = pd.read_json('https://jsonplaceholder.typicode.com/todos').groupby(['userId'])['completed'].sum().sort_values(ascending=False).to_json()


def prettyJSON(myjson):
    return json.dumps(json.loads(myjson), indent=4)


type(busyjson)

prettybusy = prettyJSON(busyjson)

with open('prettybusy.json', mode='w') as f:
    f.write(prettybusy)

# # Exercise Find Public JSON API get data and convert it into Pandas DataFrame
#
# ## Many possible sources
#
# https://github.com/toddmotto/public-apis
#     
# ### You want the ones without authorization and WITH CORS unless you are feeling adventurous and want to try with auth
#
#

df = pd.read_json('https://cat-fact.herokuapp.com/facts/random?animal_type=cat&amount=50')
df.head()

df.columns = sorted(df.columns)
df.head()

response = requests.get("https://cat-fact.herokuapp.com/facts/random?animal_type=cat&amount=50")
if response.status_code != 200:
    print("Bad Response: ", response.status_code)
print(response.status_code)
cats = json.loads(response.text)
cats[:3]

response = requests.get("https://cat-fact.herokuapp.com/facts/random",
                        params={"animal_type":"cat", "amount":20})
if response.status_code != 200:
    print("Bad Response: ", response.status_code)
print(response.status_code)
cats = json.loads(response.text) # simpler response.json() 
cats[:3]

len(cats)`

response.status_code

cats2 = response.json()
len(cats2)

cats2 == cats, cats2 is cats 
#data are the same we just have two different copies of them



df.loc[0, 'text']

# +
## For authorization you generally need some sort of token(key)
# One example for zendesk API  https://develop.zendesk.com/hc/en-us/community/posts/360001652447-API-auth-in-python


# For an API token, append '/token' to your username and use the token as the password:
## This will not work for those without zendesk access token

url = 'https://your_subdomain.zendesk.com/api/v2/users/123.json'
r = requests.get(url, auth=('<EMAIL>.com/token', 'your_token'))
# For an OAuth token, set an Authorization header:

bearer_token = 'Bearer ' + access_token
header = {'Authorization': bearer_token}
url = 'https://your_subdomain.zendesk.com/api/v2/users/123.json'
r = requests.get(url, headers=header)


# -

def myReadJSON(url):
    response = requests.get(url)
    if response.status_code != 200:
        print("Bad Response: ", response.status_code)
    print("Status CODE", response.status_code)
    return json.loads(response.text)



rawdrinks = myReadJSON("https://www.thecocktaildb.com/api/json/v1/1/search.php?s=margarita")
type(rawdrinks)

rawdrinks.keys()

mydrinks = pd.DataFrame(rawdrinks['drinks'])
mydrinks.head()

# we can Transpose to get a sense of all columns
mydrinks.head().T

drinks = pd.read_json("https://www.thecocktaildb.com/api/json/v1/1/search.php?s=margarita")
drinks.head()

# +
# requests also works with post type of requests
# -

url = "http://www.recipepuppy.com/api/?i=onions,garlic&q=omelet&p=3"

response = requests.get(url)
response.status_code

o_data = response.json()
type(o_data)

o_recipes = o_data.get('results')
type(o_recipes)

o_recipes[:3]

# for 100 suggestion is to use time.sleep(0.2)
# it is good manners to sleep a little to avoid DDOS attack on API server
import time
time.sleep(0.5) # half a second delay

StarcoderdataJupyterScriptsDedupFiltered