%matplotlib inline 
from selenium import webdriver
import time,re,json,numpy as np
import pandas as pd
from collections import defaultdict,Counter
import matplotlib.pyplot as plt

"""
Explanation: Web Scraping: Extracting Data from the Web
Some Import
End of explanation
"""


url = "http://www.imdb.com/list/ls061683439/"
with open('./img/filmfare.json',encoding="utf-8") as f:
    datatbl = json.load(f)
driver = webdriver.Chrome(datatbl['data']['chromedriver'])
driver.get(url)

"""
Explanation: Initial Setup and Launch the browser to open the URL
End of explanation
"""


from bs4 import BeautifulSoup
soup = BeautifulSoup(driver.page_source, 'lxml')
lstelem = soup.findAll("div", attrs={"class" : 'info'})
Movielist = []
for lst in lstelem:
    Movielist.append((lst.find('b').contents[0]).text)
print("First 10 Movies in the list")
Movielist[0:10]

"""
Explanation: Beautiful Soup
End of explanation
"""


def ExtractText(Xpath):
    textlist=[]
    if(Xpath=='Movies_Director_Xpath'):
        for item in range(1,123,2):
            textlist.append(driver.find_element_by_xpath(datatbl['data'][Xpath]+'[%d]'%item).text)
    else:
        [textlist.append(item.text) for item in driver.find_elements_by_xpath(datatbl['data'][Xpath])]
    return textlist

"""
Explanation: Getting Data
Function to extract the data from Web using Selenium
End of explanation
"""


#Extracting Data from Web
Movies_Votes,Movies_Name,Movies_Ratings,Movies_RunTime=[[] for i in range(4)]
datarepo = [[]]*5
Xpath_list = ['Movies_Name_Xpath','Movies_Rate_Xpath','Movies_Runtime_Xpath','Movies_Votes_Xpath',
              'Movies_Director_Xpath']

for i in range(5):
    if(i==3):
        driver.find_element_by_xpath(datatbl['data']['listview']).click()
    if(i==4):
        driver.find_element_by_xpath(datatbl['data']['detailview']).click()
    datarepo[i] = ExtractText(Xpath_list[i])

driver.quit()

"""
Explanation: Let's extract all the required data like Ratings,Votes,Genre, Year of Release for the Best Movies
End of explanation
"""


# Movie Name List & Ratings
print(datarepo[0][:5])
print(datarepo[1][:5])
print(datarepo[3][:5])
print(datarepo[4][:5])
print("")
print(datarepo[3][:5])

"""
Explanation: Data in List
End of explanation
"""


# Result in a Python Dictionary
Years=range(2015,1954,-1)
result = defaultdict(dict)
for i in range(0,len(datarepo[0])):
    result[i]['Movie Name']= datarepo[0][i]
    result[i]['Year']= Years[i]
    result[i]['Rating']= datarepo[1][i]
    result[i]['Votes']= datarepo[3][i]
    result[i]['RunTime']= datarepo[2][i]
    result[i]['Genre']= datatbl['data']['Genre'][i]
    result[i]['Director']= datarepo[4][i]


"""
Explanation: Store Data in a  Python Dictionary
End of explanation
"""


print(json.dumps((result[0]),indent=4))

print(json.dumps((result),indent=4))

"""
Explanation: Let's see now how the data in dictionary looks like?
End of explanation
"""


for key,values in result.items():
    values['Votes'] = int(values['Votes'].replace(",",""))
    values['Rating']= float(values['Rating'])
    values['Director']= values['Director'].replace('Director: ','')
    try:
        values['RunTime'] = int(re.findall(r'\d+',values['RunTime'])[-1])
    except TypeError:
        values['RunTime'] = np.NaN
    except IndexError:
        values['RunTime'] = np.NaN

"""
Explanation: Let's clean the data

Replace the comma(,) in Vote Value and change the data type to int
Change the Data type for Rating and RunTime
End of explanation
"""


print(json.dumps((result[0]),indent=4))

"""
Explanation: Now let's look at the data and see how it looks like
End of explanation
"""


print(json.dumps((result),indent=4))

"""
Explanation: Movie details in dictionary
End of explanation
"""


# create dataframe
df = pd.DataFrame.from_dict(result,orient='index')
df = df[['Year', 'Movie Name', 'Rating', 'Votes','Genre','RunTime','Director']]
df.index = np.arange(1, 62)
df.head(10)

"""
Explanation: Data in Pandas Dataframe
End of explanation
"""


df

df[df['RunTime'].isnull()==True]

nans = df.shape[0] - df.dropna().shape[0]
print('%d rows have missing values' % nans)

"""
Explanation: Let's use some of the Pandas functions now and start the Analysis
No. of rows with missing values(NaN)
End of explanation
"""


df=df.fillna(int(df['RunTime'].mean()))
df.ix[[41, 49,59]]

df.info()

"""
Explanation: Replace NaN with Mean
End of explanation
"""


#Highest Rating Movies
df1=df.sort_values('Rating',ascending=[False]).head(5)
df1.index = np.arange(1, 6)
df1

"""
Explanation: Movies with Highest Ratings
The top five movies with Maximum Rating since 1955
End of explanation
"""


df.plot(x=df.Year,y=['Rating']);


"""
Explanation: Rating Trend for Best Movies from last 65 years
End of explanation
"""


df1=df.sort_values('Rating',ascending=[True]).head(5)
df1.index = np.arange(1, 6)
df1

"""
Explanation: Movies with Lowest Ratings
End of explanation
"""


#Movies with maximum Run Time
df1=df.sort_values(['RunTime'],ascending=[False]).head(10)
df1.index = np.arange(1, 11)
df1

"""
Explanation: Movies with Maximum Run time
Top 10 movies with maximum Run time
End of explanation
"""


df.plot(x=df.Year,y=['RunTime']);

"""
Explanation: Best Movie Run time
Let's plot a graph to see the movie run time trend from 1955 thru 2015
End of explanation
"""


df['RunTime'].mean()

"""
Explanation: Mean of the Movie Run Time
End of explanation
"""


df[(df['Rating']>=7)]['Rating'].count()

"""
Explanation: Perform some analysis on the ratings of all the Best won movies

No. of Movies Greater than IMDB 7 ratings
End of explanation
"""


import seaborn as sns
sns.set_style("white")
Rating_Histdic = defaultdict(dict)

Rating_Histdic['Btwn 6&7'] = df[(df['Rating']>=6)&(df['Rating']<7)]['Rating'].count()
Rating_Histdic['GTEQ 8'] = df[(df['Rating']>=8)]['Rating'].count()
Rating_Histdic['Btwn 7 & 8'] = df[(df['Rating']>=7)&(df['Rating']<8)]['Rating'].count()


plt.bar(range(len(Rating_Histdic)), Rating_Histdic.values(), align='center',color='b',width=0.4)
plt.xticks(range(len(Rating_Histdic)), Rating_Histdic.keys(), rotation=25);

"""
Explanation: Movie Ratings Visualization using Bar Graph
End of explanation
"""


sns.set_style("white")
sns.set_context("notebook")

Rating_Hist = []
import numpy as np
    
Rating_Hist.append(Rating_Histdic['Btwn 6&7'])
Rating_Hist.append(Rating_Histdic['GTEQ 8'])
Rating_Hist.append(Rating_Histdic['Btwn 7 & 8'])

labels = ['Btwn 6&7', 'GTEQ 8', 'Btwn 7 & 8']
colors = ['red', 'c', 'lightgreen']

plt.pie(Rating_Hist,labels=labels, colors=colors,autopct='%1.1f%%', shadow=True, startangle=90);
plt.figure(figsize=(8, 6));

"""
Explanation: Percentage distribution of the Ratings in a Pie-Chart
End of explanation
"""


Category=Counter(datatbl['data']['Genre'])
df1 = pd.DataFrame.from_dict(Category,orient='index')
df1 = df1.sort_values([0],ascending=[False]).head(5)
df1.plot(kind='barh',color=['g','c','m']);

"""
Explanation: Best Picture by Genre
Let's analyze the Genre for the best won movies
End of explanation
"""


df['freq']= df.groupby('Director')['Director'].transform('count')
df2=df[df['freq']>1]
del df2['freq']

df2.groupby(['Director','Year', 'Movie Name',
             'Rating', 'Genre','Votes','RunTime']).count()[0:100]

"""
Explanation: Directors whose movie won more than once for Best Film
Movies for which they won best film award
End of explanation
"""


