# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python [conda root]
#     language: python
#     name: conda-root-py
# ---

import time
import logging
import os
import json
import requests
import pandas as pd
from s3fs.core import S3FileSystem 

# +
os.environ['AWS_CONFIG_FILE'] = 'aws_config.ini'

s3 = S3FileSystem(anon=False)
key = 'TheNumbers_budgets.csv'
bucket = 'movie-torrents'

df = pd.read_csv(s3.open('{}/{}'.format(bucket, key), mode='rb'), index_col=0)
df.head()

# +
logger = logging.getLogger('OMDB_API')
logger.setLevel(logging.INFO)

# create a file handler
handler = logging.FileHandler('omdb_api.log')
handler.setLevel(logging.INFO)

# create a logging format
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
# -

# tupple of movie, title
title = df['title']
year = [year[:4] for year in df['release_date']]
movie_tup = [(title, year) for title, year in zip(title, year)]

# +
with open('./scripts/omdb_api.key', 'r') as read_file:
    omdb_api_key = read_file.read().strip()

lst = []

for title, year in movie_tup:
    # meter number of requests to omdb api
    time.sleep(0.5)

    # omdb api address
    payload = {'t': title, 'y': year, 'apikey': omdb_api_key}
    html = requests.get('http://www.omdbapi.com', params=payload)

    # check for 200 code (good)
    resp = json.loads(html.text)
    if html.status_code != 200 or 'Error' in resp.keys():
        logger.info('Year:{0} - Title:{1}'.format(year, title))
        continue

    html_text = html.text
    html_json = json.loads(html_text)
    
    lst.append(html_json)
    
    if len(lst) > 20:
        break
# -

df = pd.DataFrame.from_dict(lst, orient='columns')
df = df[['Actors', 'Awards', 'BoxOffice', 'Country', 'DVD', 'Director', 'Genre',
       'Language', 'Metascore', 'Production', 'Rated', 'Released', 'Runtime', 'Title', 
       'Type', 'Writer', 'imdbID', 'imdbRating', 'imdbVotes']]

# +
for col in ['BoxOffice', 'imdbVotes']:
    df[col].replace(to_replace='N/A', value='0', inplace=True)
    df[col] = df[col].replace(r'[\$,]', '', regex=True).astype(int)

df['Runtime'] = df['Runtime'].replace(r'[ min]', '', regex=True).astype(int)

for col in ['DVD', 'Released']:
    df[col] = pd.to_datetime(df[col], errors='coerce', format='%d %b %Y')

for col in ['Metascore', 'imdbRating']:
    df[col].replace(to_replace='N/A', value='0', inplace=True)
    df[col] = df[col].astype(float)
# -

df.head()

StarcoderdataJupyterScriptsDedupFiltered