# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] toc=true
# <h1>Table of Contents<span class="tocSkip"></span></h1>
# <div class="toc"><ul class="toc-item"><li><span><a href="#Import-the-json-and-pprint-libraries" data-toc-modified-id="Import-the-json-and-pprint-libraries-1"><span class="toc-item-num">1&nbsp;&nbsp;</span>Import the json and pprint libraries</a></span></li><li><span><a href="#Load-the-JSON-data-and-look-for-potential-issues" data-toc-modified-id="Load-the-JSON-data-and-look-for-potential-issues-2"><span class="toc-item-num">2&nbsp;&nbsp;</span>Load the JSON data and look for potential issues</a></span></li><li><span><a href="#Check-for-differences-in-the-structure-of-the-dictionaries" data-toc-modified-id="Check-for-differences-in-the-structure-of-the-dictionaries-3"><span class="toc-item-num">3&nbsp;&nbsp;</span>Check for differences in the structure of the dictionaries</a></span></li><li><span><a href="#Generate-counts-from-the-JSON-data" data-toc-modified-id="Generate-counts-from-the-JSON-data-4"><span class="toc-item-num">4&nbsp;&nbsp;</span>Generate counts from the JSON data</a></span></li><li><span><a href="#Get-the-source-data-and-confirm-that-it-has-the-anticipated-length" data-toc-modified-id="Get-the-source-data-and-confirm-that-it-has-the-anticipated-length-5"><span class="toc-item-num">5&nbsp;&nbsp;</span>Get the source data and confirm that it has the anticipated length</a></span></li><li><span><a href="#Fix-any-errors-in-the-values-in-the-dictionary" data-toc-modified-id="Fix-any-errors-in-the-values-in-the-dictionary-6"><span class="toc-item-num">6&nbsp;&nbsp;</span>Fix any errors in the values in the dictionary</a></span></li><li><span><a href="#Create-a-pandas-DataFrame" data-toc-modified-id="Create-a-pandas-DataFrame-7"><span class="toc-item-num">7&nbsp;&nbsp;</span>Create a pandas DataFrame</a></span></li><li><span><a href="#Confirm-that-we-are-getting-the-expected-values-for-source" data-toc-modified-id="Confirm-that-we-are-getting-the-expected-values-for-source-8"><span class="toc-item-num">8&nbsp;&nbsp;</span>Confirm that we are getting the expected values for source</a></span></li></ul></div>
# -

# # Import the json and pprint libraries

import pandas as pd
import numpy as np
import json
import pprint
from collections import Counter

# +
import watermark

# %load_ext watermark
# %watermark -n -v -iv
# -

# # Load the JSON data and look for potential issues

with open('data/allcandidatenewssample.json') as f:
    candidatenews = json.load(f)

len(candidatenews)

pprint.pprint(candidatenews[0:2])

pprint.pprint(candidatenews[0]['source'])

# # Check for differences in the structure of the dictionaries

Counter([len(item) for item in candidatenews])

pprint.pprint(next(item for item in candidatenews if len(item) < 9))

# checking the usage of next
pprint.pprint((item for item in candidatenews if len(item) < 9))

pprint.pprint(next(item for item in candidatenews if len(item) > 9))

pprint.pprint([item for item in candidatenews if len(item) == 2][0:10])

candidatenews = [item for item in candidatenews if len(item) > 2]

len(candidatenews)

# # Generate counts from the JSON data

politico = [item for item in candidatenews if item['source'] == "Politico"]

len(politico)

pprint.pprint(politico[0:2])

# # Get the source data and confirm that it has the anticipated length

sources = [item.get('source') for item in candidatenews]

type(sources)

len(sources)

sources[0:5]

pprint.pprint(Counter(sources).most_common(10))

# # Fix any errors in the values in the dictionary

for newsdict in candidatenews:
    newsdict.update((k, 'The Hill') for k, v in newsdict.items()
                    if k == 'source' and v == 'TheHill')

# +
# Usage of  item.get('source') instead of item['source']. 
# This is handy when there might be missing keys in a dictionary. get returns None when the key
# is missing, but we can use an optional second argument to specify a value to return.

sources = [item.get('source') for item in candidatenews]
# -

pprint.pprint(Counter(sources).most_common(10))

# # Create a pandas DataFrame

candidatenewsdf = pd.DataFrame(candidatenews)

candidatenewsdf.dtypes

# # Confirm that we are getting the expected values for source

candidatenewsdf.rename(columns={'date': 'storydate'}, inplace=True)

candidatenewsdf['storydate'] = candidatenewsdf['storydate'].astype(
    'datetime64[ns]')

candidatenewsdf.shape

candidatenewsdf['source'].value_counts(sort=True).head(10)



StarcoderdataJupyterScriptsDedupFiltered