# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/Edward-TL/amazon_scraper/blob/master/time_standarizer.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + id="P6HgF123wCY6" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 122} outputId="4f03757a-71b5-4ce8-e113-b7a50f67e0e1"
from google.colab import drive
drive.mount('/content/drive')

# + [markdown] id="RzmgSzTDwhUX" colab_type="text"
# ##Imports

# + id="O5mH3Qu0wTAK" colab_type="code" colab={}
#For the system
import os

#Manage of time
from datetime import datetime, timedelta
from pytz import timezone
import time

#Manage of files
import pandas as pd
import csv

#scrap
from bs4 import BeautifulSoup
from openpyxl.workbook import Workbook
import requests

# + [markdown] id="em6mFJ7hwiar" colab_type="text"
# #Manejador de todo

# + id="f-neKLoSwlqE" colab_type="code" colab={}


country = 'br master db'
dir = f'{country}/parquet'
databases = os.listdir(dir)
n_databases = len(databases)

#initialize
df = [None]*n_databases
largest_len_db = 0
n = 0

for file_name in databases:
    db = f'{dir}/{file_name}'
    dataframe = pd.read_parquet(db)
    df[n] = db
    if len(file_name) > largest_len_db:
        largest_len_db = len(file_name)
    n = n + 1

#Print tools
centered_db = centered_len(largest_len_db)
centered_header = centered_db('masterdbs')
dashes = '-'*largest_len_db
print(f' {centered_header} | Edition')
print(f' {dashes} | ------ ')

for d in range(len(df)):
    #file name extract
    dir_len = len(dir)+1
    parquet_len = len('.parquet')
    file_name_len = len(df[d])-parquet_len
    file_name = df[d][dir_len:file_name_len]
    #File name print to log
    print(f' {centered_db(file_name)}', end=' | ')

    #--------Edition--------
    #Load Dataframe
    edition = pd.read_parquet(df[d])
    #Ranking
    try:
        edition['Rank'] = edition['Rank'].str.extract(r'(\d+)')
        #Fechas
        edition['time'] = pd.to_datetime(edition['time'])
        edition['time'] = edition['time'] - timedelta(hours=5)

        #--------Upadting--------
        #File names
        csv_file = f'{country}/csv/{file_name}.csv'
        excel_file = f'{country}/excel/{file_name}.xlsx'
        parquet_file = f'{country}/parquet/{file_name}.parquet'
        #File updating
        edition.to_csv(csv_file,sep="|")
        edition.to_excel(excel_file, index=False)
        edition.to_parquet(parquet_file, allow_truncated_timestamps=True)

        print('Done')
    except AttributeError:
        csv_file = f'{country}/csv/{file_name}.csv'
        excel_file = f'{country}/excel/{file_name}.xlsx'
        parquet_file = f'{country}/parquet/{file_name}.parquet'

        os.remove(csv_file)
        os.remove(excel_file)
        os.remove(parquet_file)

        print('Deleted. File Error: No data in')

# + [markdown] id="nwnHBEOawncd" colab_type="text"
# #Labs

# + id="HDFCM11exg38" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 51} outputId="1416c99c-4673-4f36-e6a5-bca80fe97a83"
# %cd 'drive/My Drive/Colab Notebooks/4SS/4SS_db'
# !ls

# + id="B911g857xn6f" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 51} outputId="804e9bdd-6c2e-4536-d6d4-b45cac2ac5fb"
# %cd 'testing/Masters'
# !ls

# + id="JZ8fdxSbwluc" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 54} outputId="6e648784-49bb-4e0f-d1ef-024425564689"
country = 'mx'
file_name = 'mx-master_db_music.csv'
entry = f'{country}/csv/{file_name}'
dates = [None]*6000
r = 0

with open(entry, mode="r", encoding='utf-8') as csv_file:
    csv_reader = csv.reader(csv_file, delimiter='|')
    header = next(csv_reader)
    for row in csv_reader:
        date_str = row[1]
        date = datetime.strptime(date_str, '%Y-%m-%d %H:%M:%S.%f')
        date = date.strftime('%Y-%m-%d %H:%M')
        dates[r] = date
        r = r + 1

print(dates)


# + [markdown] id="ZaLJFOrj2cS4" colab_type="text"
# #Time standarizer

# + id="KyiyIutu2eEv" colab_type="code" colab={}
def standarize_dates(date):
    try:
        date = datetime.strptime(str(date), '%Y-%m-%d %H:%M:%S.%f')
    except ValueError:
        try:
            date = datetime.strptime(str(date), '%Y-%m-%d %H:%M:%S')
        except:
            date = datetime.strptime(str(date), '%Y-%m-%d %H_%M')

    date = date.strftime('%Y-%m-%d %H:%M')
    
    return date

StarcoderdataJupyterScriptsDedupFiltered