# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] pycharm={"name": "#%% md\n"}
# # Clean and explore galaxy zoo datasets
# This file details the data cleansing of the Galaxy Zoo dataset and explains the correlations found between the datasets features.
# <NAME>, <NAME>

# + [markdown] pycharm={"name": "#%% md\n"}
#   <img src="../Pics/AM0002.png" width="900"> </br>
# Picture of AM0002 captured by Hubble telescope

# + [markdown] pycharm={"name": "#%% md\n"}
# ## Abstract
#
# ### Galaxy zoo
# ...
#
# ### Galaxies classification
# ...

# + [markdown] pycharm={"name": "#%% md\n"}
# ## Pre-processing
# ....
# ### Library imports

# + pycharm={"name": "#%%\n"}
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

import warnings
warnings.filterwarnings('ignore')

# + [markdown] pycharm={"name": "#%% md\n"}
# ### Basic analysis

# + pycharm={"name": "#%%\n"}
from IPython.core.display import display

def checkIfHasRowIncompatible(dataset, rowName1, rowName2):
    return dataset[dataset[rowName1] == 1][dataset[rowName2] == 1].shape[0] > 0

def VisualiseDataset(dataset):

    print("En-tête du dataset :\n----------\n")
    display(dataset.head())

    print("Informations des types du dataset :\n----------\n")
    display(dataset.info())

    print("\n----------\nTaille du dataset :")
    display(dataset.shape)

    print("Informations du dataset :\n----------\n")
    display(dataset.describe())

    print("Pourcentage de valeurs manquantes :\n----------\n")
    display((df.isna().sum()/df.shape[0]).sort_values())

    print("Vérification si valeurs multiple à 1 incompatible :\n----------\n")
    print("A ligne Spirale == Elliptique : {}\n".format(checkIfHasRowIncompatible(df, "SPIRAL", "ELLIPTICAL")))
    print("A ligne Spirale == Incertaine : {}\n".format(checkIfHasRowIncompatible(df, "SPIRAL", "UNCERTAIN")))
    print("A ligne Elliptique == Incertaine : {}\n".format(checkIfHasRowIncompatible(df, "ELLIPTICAL", "UNCERTAIN")))

    print("Type des valeurs :\n----------\n")
    display(df.dtypes.value_counts().plot.pie())


# + pycharm={"name": "#%%\n"}
data = pd.read_csv("../../Datas/GalaxyZoo1_DR_table2.csv")
df = data.copy()
VisualiseDataset(df)

# + [markdown] pycharm={"name": "#%% md\n"}
# '### Basic checklist
# - **Targets variables** : Spiral, Elliptical and Uncertain
# - **lines and columns** : (667944, 16)
# - **Type de variables** : Majortiée de numérique, le reste sont objet (ici des dates)
# - **Analyse des valeurs manquantes** : Aucunes valeurs manquantes

# + pycharm={"name": "#%%\n"}
dfbiased = df.copy()
dfbiased.drop(['OBJID', 'P_CS', 'P_EL', 'P_CW', 'P_MG', 'P_ACW', 'P_EDGE', 'RA', 'DEC', 'ELLIPTICAL', 'SPIRAL', 'UNCERTAIN'], axis=1, inplace=True)
sns.histplot(dfbiased['NVOTE'])

for col in dfbiased.select_dtypes('double'):
    plt.figure()
    sns.distplot(df[col])


# + [markdown] pycharm={"name": "#%% md\n"}
# Supression des valeurs ou le nombre de vote est inférieur à 50.

# + pycharm={"name": "#%%\n"}
shapeBase = len(df.index)
df = df[df.NVOTE >= 15]
df = df[df.P_DK > 0.05]
shapeAfterModif = len(df.index)

print("Nombre de ligne suprimées : ")
print(shapeBase - shapeAfterModif)

df.shape

# + pycharm={"name": "#%%\n"}
correlationDf = df.copy()
correlationDf.drop(['OBJID', 'NVOTE', 'DEC', 'RA', 'SPIRAL', 'UNCERTAIN', 'ELLIPTICAL'], axis=1, inplace=True)

# + pycharm={"name": "#%%\n"}
sns.clustermap(correlationDf.corr())

# + [markdown] pycharm={"name": "#%% md\n"}
# Création d'un dataset final

# + pycharm={"name": "#%%\n"}
print("Nombre de galaxies ayant le statut MERGED :", df["P_MG"][df["P_MG"] > 0.4].count())

# + pycharm={"name": "#%%\n"}
dfWithPreProcess = df.copy()
dfWithPreProcess.drop(['NVOTE', 'P_EL', 'P_CW', 'P_ACW', 'P_EDGE', 'P_DK', 'P_CS', 'P_EL_DEBIASED', 'P_CS_DEBIASED'], axis=1, inplace=True)

# + pycharm={"name": "#%%\n"}
dfWithPreProcess['MERGED'] = np.where(df['P_MG'] > 0.5, 1, 0)
dfWithPreProcess.head()

# + pycharm={"name": "#%%\n"}
#Merge the three columns by using their name as type in the column "TYPE".
columns = ["MERGED", "SPIRAL", "ELLIPTICAL", "UNCERTAIN"]
for name in columns: dfWithPreProcess.loc[dfWithPreProcess[name]==1, 'TYPE'] = name
#Drop all columns not needed and each rows with type "UNCERTAIN"
dfWithPreProcess.drop(index=dfWithPreProcess[dfWithPreProcess['TYPE'] == "UNCERTAIN"].index,
    columns=['P_MG', 'UNCERTAIN', "MERGED", "ELLIPTICAL", "SPIRAL"], inplace=True)
dfWithPreProcess.to_csv('../Datas/dataWithPreProcess.csv', index=False)
dfWithPreProcess.head()

# + pycharm={"name": "#%%\n"}
dfShow = dfWithPreProcess[dfWithPreProcess["TYPE"] == "MERGED"]
dfShow.head()

# + pycharm={"name": "#%%\n"}
dfShow = dfWithPreProcess[dfWithPreProcess["TYPE"] == "ELLIPTICAL"]
dfShow.head()

# + pycharm={"name": "#%%\n"}
dfShow = dfWithPreProcess[dfWithPreProcess["TYPE"] == "SPIRAL"]
dfShow.head()

# + pycharm={"name": "#%%\n"}
dfShow = dfWithPreProcess[dfWithPreProcess["TYPE"] == "UNKNOWN"]
dfShow.head()

StarcoderdataJupyterScriptsDedupFiltered