# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# +
# Function that returns list of bow for all movies in a given set(train/dev/test) of the given file root

from collections import defaultdict

def bow_movie(root,start,end):
    bow_per_movie = [] # list of dict where each element of bow_per_movie is bow for that movie
    for i in range(start,end):
        bow = defaultdict(float)
        string = ""
        for j in range(1,len(root[i])):
            string += root[i][j].text

        tokens =string.split()
        l_tokens = map(lambda t: t.lower(), tokens)

        for token in l_tokens:
            bow[token] += 1.0
        bow_per_movie.append(bow)
    return bow_per_movie



# -

# Function that returns vocab for all movies in the training set of given file root
def vocab(bow_per_movie):
    vocab = defaultdict(float)
    
    stop_words=["a","an", "and","are","as","at","be","by","for","from","has","he", "in","is","it","its","of","on","that","the","to","was","were","will","with"]
    
    for bow in bow_per_movie:
        for key,value in bow.iteritems():
            vocab[key] += value
            
    ## removing stop words
    #for key,value in vocab.items():
    #    if key in stop_words:
    #        vocab.pop(key)
            
    ## removing infrequent words with frequency less than 5
    #for key,value in vocab.items():
    #    if value<5:
    #        vocab.pop(key)
    
    return vocab


# Function that returns feature vector for all movies in the given set(train/dev/test) of given file root
def fvec(bow_per_movie,vocab):
    fvec_per_movie = [] # list of lists where each element of fvec_per_movie is a feature vector for that movie

    for bow in bow_per_movie:
        fvec = []
        for key,value in vocab.iteritems():
            if key in bow:
                fvec.append(value)
            else:
                fvec.append(0)

        fvec_per_movie.append(fvec)
    return fvec_per_movie


# Function that returns a list of target variables i.e. revenue for all movies in the given set(train/dev/test) of given file root
def true_rev(start,end,root):
    rev = []
    for i in range(start,end):
        rev.append(root[i][0].attrib['yvalue'])
    rev=np.array(rev).astype(np.float)
    return rev


# # Predicting revenue in training set

# +
from sklearn.linear_model import Ridge
from sklearn import linear_model
from sklearn.tree import DecisionTreeRegressor
from sklearn.model_selection import GridSearchCV
import scipy
import numpy as np

def gsv(train_x,train_y):
    parameters = {'max_depth':[10, 1000]}
    clf = GridSearchCV(DecisionTreeRegressor(), parameters)
    clf.fit(train_x, train_y)
    return clf

def train(train_x,train_y):
    #clf = Ridge(alpha=0)
    clf=linear_model.Lasso(alpha=0)
    #clf=DecisionTreeRegressor(max_depth=10)
    clf.fit(train_x, train_y)
    return clf

def predict(clf, test_x): 
    return clf.predict(test_x)

def cal_mae(y_hat,y):
    return np.mean(abs(y_hat-y))


# -

import xml.etree.ElementTree as ET
tree = ET.parse('dataset\\movies-data-v1.0\\movies-data-v1.0\\perscreen-7domains-train-dev.tl.xml')
root_traindev_ps = tree.getroot()

import numpy as np
#bow_per_movie_train_ps = bow_movie(root_traindev_ps,0,1147)
true_rev_train_ps = true_rev(0,1147,root_traindev_ps)

# +
import pickle

f = open('train_y_ps.txt', 'w')
pickle.dump(true_rev_train_ps , f)
f.close()

# -

# only when changing vocab 
#####################################################################################3
vocab_ps=vocab(bow_per_movie_train_ps)
fvec_train_ps = fvec(bow_per_movie_train_ps,vocab_ps)

trained_classifier=train(fvec_train_ps,true_rev_train_ps)

# # Predicting revenue in development set

bow_per_movie_dev_ps = bow_movie(root_traindev_ps,1147,1464)

#fvec_dev_ps = fvec(bow_per_movie_dev_ps,vocab_ps)
true_rev_dev_ps = true_rev(1147,1464,root_traindev_ps)

# +
import pickle

f = open('dev_y_ps.txt', 'w')
pickle.dump(true_rev_dev_ps, f)
f.close()


# +
#############################
# insert prediction code here
pred_rev_dev_ps = predict(trained_classifier,fvec_dev_ps) 
mae_rev_dev_ps= cal_mae(pred_rev_dev_ps,true_rev_dev_ps)
############################

print "The mean absolute error for dev set is ", mae_rev_dev_ps
# -

# # Predicting revenue in test set

import xml.etree.ElementTree as ET
tree = ET.parse('dataset\\movies-data-v1.0\\movies-data-v1.0\\perscreen-7domains-train-test.tl.xml')
root_traintest_ps = tree.getroot()

bow_per_movie_test_ps = bow_movie(root_traintest_ps,1147,1401)
fvec_test_ps = fvec(bow_per_movie_test_ps,vocab_ps)
true_rev_test_ps = true_rev(1147,1401,root_traintest_ps)
#########################
# insert prediction code here
# pred_rev_test_ps =
#########################



StarcoderdataJupyterScriptsDedupFiltered