# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import requests
from bs4 import BeautifulSoup
import tqdm.notebook as tq
import pickle
import pandas as pd


def get_pages(page_url):
    headers = {
        'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36'
    }
    response = requests.get(url=page_url,headers=headers)
    page_soup = BeautifulSoup(response.text,'lxml')
    return page_soup


urls = ['https://www.transfermarkt.com/arsenal-fc/startseite/verein/11',
        'https://www.transfermarkt.com/fc-chelsea/startseite/verein/631',
        'https://www.transfermarkt.com/fc-liverpool/startseite/verein/31',
        'https://www.transfermarkt.com/manchester-city/startseite/verein/281',
        'https://www.transfermarkt.com/manchester-united/startseite/verein/985',
        'https://www.transfermarkt.com/tottenham-hotspur/startseite/verein/148',
        'https://www.transfermarkt.com/atletico-madrid/startseite/verein/13',
        'https://www.transfermarkt.com/fc-barcelona/startseite/verein/131',
        'https://www.transfermarkt.com/real-madrid/startseite/verein/418',
        'https://www.transfermarkt.com/fc-paris-saint-germain/startseite/verein/583',
        'https://www.transfermarkt.com/fc-bayern-munchen/startseite/verein/27',
        'https://www.transfermarkt.com/borussia-dortmund/startseite/verein/16',
        'https://www.transfermarkt.com/inter-mailand/startseite/verein/46',
        'https://www.transfermarkt.com/ac-mailand/startseite/verein/5',
        'https://www.transfermarkt.com/juventus-turin/startseite/verein/506',
        'https://www.transfermarkt.com/as-rom/startseite/verein/12'
       ]


premier_urls = ['https://www.transfermarkt.com/arsenal-fc/startseite/verein/11',
                'https://www.transfermarkt.com/fc-chelsea/startseite/verein/631',
                'https://www.transfermarkt.com/fc-liverpool/startseite/verein/31',
                'https://www.transfermarkt.com/manchester-city/startseite/verein/281',
                'https://www.transfermarkt.com/manchester-united/startseite/verein/985',
                'https://www.transfermarkt.com/tottenham-hotspur/startseite/verein/148',
                'https://www.transfermarkt.com/aston-villa/startseite/verein/405',
                'https://www.transfermarkt.com/brighton-amp-hove-albion/startseite/verein/1237',
                'https://www.transfermarkt.com/fc-burnley/startseite/verein/1132',
                'https://www.transfermarkt.com/crystal-palace/startseite/verein/873',
                'https://www.transfermarkt.com/fc-everton/startseite/verein/29',
                'https://www.transfermarkt.com/leeds-united/startseite/verein/399',
                'https://www.transfermarkt.com/leicester-city/startseite/verein/1003',
                'https://www.transfermarkt.com/newcastle-united/startseite/verein/762',
                'https://www.transfermarkt.com/fc-southampton/startseite/verein/180',
                'https://www.transfermarkt.com/west-ham-united/startseite/verein/379',
                'https://www.transfermarkt.com/wolverhampton-wanderers/startseite/verein/543',
                'https://www.transfermarkt.com/fc-fulham/startseite/verein/931',
                'https://www.transfermarkt.com/west-bromwich-albion/startseite/verein/984',
                'https://www.transfermarkt.com/sheffield-united/startseite/verein/350'
        ]


def get_names(url):
    soup = get_pages(url)
    hide = soup.find_all('td',class_='hide')
    for i in range(len(hide)):
        name = hide[i].text
        if name not in names:
            names.append(name)


names = []
for url in tq.tqdm(premier_urls):
    url1 = url+'?saison_id=2020'
    url2 = url+'?saison_id=2019'
    get_names(url)
    get_names(url1)
    get_names(url2)


def process(names):
    result = []
    for name in names:
        sp = name.split()
        for word in sp:
            word = word.lower()
            if word not in result:
                result.append(word)
    return result


player_names = process(names)
pickle.dump(player_names, open('../data/pretrained/premier_player_names.pkl', 'wb'))

coach_names = ['carlo','ancelotti','diego','simeone','julen', 'lopetegui','xavi','ronald','koeman',
               '<NAME>','<NAME>','<NAME>','<NAME>',
               '<NAME>','<NAME>','<NAME>','<NAME>','<NAME>','<NAME>',
               '<NAME>','<NAME>','<NAME>','<NAME>','<NAME>',
               '<NAME>','<NAME>','<NAME>','<NAME>','<NAME>','<NAME>',
               '<NAME>','<NAME>','<NAME>','<NAME>','<NAME>','<NAME>',
               '<NAME>','<NAME>','<NAME>','<NAME>','mikel','Arteta','<NAME>','<NAME>',
               '<NAME>','<NAME>','<NAME>','<NAME>','<NAME>','<NAME>','<NAME>','<NAME>',
               '<NAME>','<NAME>','<NAME>','<NAME>','<NAME>'
    ]

manager_names = process(coach_names)
pickle.dump(manager_names, open('../data/pretrained/manager_names.pkl', 'wb'))

stadium_names = ['Emirates','Stamford Bridge','Anfield','Etihad','Trafford','Westfalenstadion','Allianz','<NAME>'
                 '<NAME>','<NAME>','Olimpico','Parc des Princes']

stadium = process(stadium_names)
pickle.dump(stadium, open('../data/pretrained/stadium_names.pkl', 'wb'))

game_url = '../data/match_info.html'
soup = BeautifulSoup(open(game_url,'r',encoding = 'utf-8'))
game_time = soup.find_all('span',class_='matches__date')
team = soup.find_all('span',class_='swap-text__target')[1:]
game_times = pd.DataFrame()
for i in range(380):
    game_times = game_times.append({
        'time' : game_time[i].text.split()[0],
        'team1' : team[2*i].text,
        'team2' : team[2*i+1].text
    },ignore_index=True)

game_time = pd.read_csv('../data/utils/game_time.csv')
game_time

game_time['time'] = pd.to_datetime(game_time.time)
game_time

game_time.iloc[0]['time']

from datetime import datetime,timedelta
time = []
for i in range(game_time.shape[0]):
    time.append(game_time.iloc[i]['time']-timedelta(hours=1))
game_time['time']=time
game_time

game_time['time'] = pd.to_datetime(game_time['time']).dt.time

game_time

game_time.to_csv('../data/utils/game_time.csv',index=False)

type(soup)

game_times[:20]



StarcoderdataJupyterScriptsDedupFiltered