# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
# %matplotlib inline
# %config InlineBackend.figure_formats = {'png', 'retina'}

data_key = pd.read_csv('key.csv')
data_key = data_key[data_key['station_nbr'] != 5]
data_weather = pd.read_csv('weather.csv')
data_weather = data_weather[data_weather['station_nbr'] != 5] ## Station 5번 제거한 나머지
data_train = pd.read_csv('train.csv')

df = pd.merge(data_weather, data_key)
station_nbr = df['station_nbr']
df.drop('station_nbr', axis=1, inplace=True)
df['station_nbr'] = station_nbr
df = pd.merge(df, data_train)
df.head() # Station 5번을 뺀 나머지 Merge 완성
# -

# 'M'과 '-'을 np.nan으로 값을 변경하기 전에, '  T'값을 먼저 snowfall=0.05, preciptotal = 0.005로 변경하자 
df['snowfall'][df['snowfall'] == '  T'] = 0.05
df['preciptotal'][df['preciptotal'] == '  T'] = 0.005
df['snowfall'][df['snowfall'] == '  T'], df['preciptotal'][df['preciptotal'] == '  T']
# T 값 변경 완료. 이제, 19개 Station 별로 정리하기 (5번 Station 생략)

df_s_1 = df[df['station_nbr'] == 1]; df_s_8 = df[df['station_nbr'] == 8]; df_s_15 = df[df['station_nbr'] == 15]
df_s_2 = df[df['station_nbr'] == 2]; df_s_9 = df[df['station_nbr'] == 9]; df_s_16 = df[df['station_nbr'] == 16]
df_s_3 = df[df['station_nbr'] == 3]; df_s_10 = df[df['station_nbr'] == 10]; df_s_17 = df[df['station_nbr'] == 17]
df_s_4 = df[df['station_nbr'] == 4]; df_s_11 = df[df['station_nbr'] == 11]; df_s_18 = df[df['station_nbr'] == 18]
df_s_5 = df[df['station_nbr'] == 5]; df_s_12 = df[df['station_nbr'] == 12]; df_s_19 = df[df['station_nbr'] == 19]
df_s_6 = df[df['station_nbr'] == 6]; df_s_13 = df[df['station_nbr'] == 13]; df_s_20 = df[df['station_nbr'] == 20]
df_s_7 = df[df['station_nbr'] == 7]; df_s_14 = df[df['station_nbr'] == 14]

# +
# Each Station tavg 의 M값을 np.nan으로 변경
df_s_1_tavg = df_s_1['tavg'].copy(); df_s_1_tavg = pd.to_numeric(df_s_1_tavg, errors = 'coerce')
df_s_2_tavg = df_s_2['tavg'].copy(); df_s_2_tavg = pd.to_numeric(df_s_2_tavg, errors = 'coerce')
df_s_3_tavg = df_s_3['tavg'].copy(); df_s_3_tavg = pd.to_numeric(df_s_3_tavg, errors = 'coerce')
df_s_4_tavg = df_s_4['tavg'].copy(); df_s_4_tavg = pd.to_numeric(df_s_4_tavg, errors = 'coerce')
df_s_5_tavg = df_s_5['tavg'].copy(); df_s_5_tavg = pd.to_numeric(df_s_5_tavg, errors = 'coerce')
df_s_6_tavg = df_s_6['tavg'].copy(); df_s_6_tavg = pd.to_numeric(df_s_6_tavg, errors = 'coerce')
df_s_7_tavg = df_s_7['tavg'].copy(); df_s_7_tavg = pd.to_numeric(df_s_7_tavg, errors = 'coerce')
df_s_8_tavg = df_s_8['tavg'].copy(); df_s_8_tavg = pd.to_numeric(df_s_8_tavg, errors = 'coerce')
df_s_9_tavg = df_s_9['tavg'].copy(); df_s_9_tavg = pd.to_numeric(df_s_9_tavg, errors = 'coerce')
df_s_10_tavg = df_s_10['tavg'].copy(); df_s_10_tavg = pd.to_numeric(df_s_10_tavg, errors = 'coerce')
df_s_11_tavg = df_s_11['tavg'].copy(); df_s_11_tavg = pd.to_numeric(df_s_11_tavg, errors = 'coerce')
df_s_12_tavg = df_s_12['tavg'].copy(); df_s_12_tavg = pd.to_numeric(df_s_12_tavg, errors = 'coerce')
df_s_13_tavg = df_s_13['tavg'].copy(); df_s_13_tavg = pd.to_numeric(df_s_13_tavg, errors = 'coerce')
df_s_14_tavg = df_s_14['tavg'].copy(); df_s_14_tavg = pd.to_numeric(df_s_14_tavg, errors = 'coerce')
df_s_15_tavg = df_s_15['tavg'].copy(); df_s_15_tavg = pd.to_numeric(df_s_15_tavg, errors = 'coerce')
df_s_16_tavg = df_s_16['tavg'].copy(); df_s_16_tavg = pd.to_numeric(df_s_16_tavg, errors = 'coerce')
df_s_17_tavg = df_s_17['tavg'].copy(); df_s_17_tavg = pd.to_numeric(df_s_17_tavg, errors = 'coerce')
df_s_18_tavg = df_s_18['tavg'].copy(); df_s_18_tavg = pd.to_numeric(df_s_18_tavg, errors = 'coerce')
df_s_19_tavg = df_s_19['tavg'].copy(); df_s_19_tavg = pd.to_numeric(df_s_19_tavg, errors = 'coerce')
df_s_20_tavg = df_s_20['tavg'].copy(); df_s_20_tavg = pd.to_numeric(df_s_20_tavg, errors = 'coerce')

# 각 각의 Station의 Nan 값에, 위에서 구한 각각 station의 평균 값을 넣어서 NaN 값 뺏을 때와 비교할 것.
df_s_1_tavg_with_mean = df_s_1_tavg.copy(); df_s_1_tavg_with_mean[df_s_1_tavg_with_mean.isnull()] = df_s_1_tavg.mean()
df_s_2_tavg_with_mean = df_s_2_tavg.copy(); df_s_2_tavg_with_mean[df_s_2_tavg_with_mean.isnull()] = df_s_2_tavg.mean()
df_s_3_tavg_with_mean = df_s_3_tavg.copy(); df_s_3_tavg_with_mean[df_s_3_tavg_with_mean.isnull()] = df_s_3_tavg.mean()
df_s_4_tavg_with_mean = df_s_4_tavg.copy(); df_s_4_tavg_with_mean[df_s_4_tavg_with_mean.isnull()] = df_s_4_tavg.mean()
df_s_5_tavg_with_mean = df_s_5_tavg.copy(); df_s_5_tavg_with_mean[df_s_5_tavg_with_mean.isnull()] = df_s_5_tavg.mean()
df_s_6_tavg_with_mean = df_s_6_tavg.copy(); df_s_6_tavg_with_mean[df_s_6_tavg_with_mean.isnull()] = df_s_6_tavg.mean()
df_s_7_tavg_with_mean = df_s_7_tavg.copy(); df_s_7_tavg_with_mean[df_s_7_tavg_with_mean.isnull()] = df_s_7_tavg.mean()
df_s_8_tavg_with_mean = df_s_8_tavg.copy(); df_s_8_tavg_with_mean[df_s_8_tavg_with_mean.isnull()] = df_s_8_tavg.mean()
df_s_9_tavg_with_mean = df_s_9_tavg.copy(); df_s_9_tavg_with_mean[df_s_9_tavg_with_mean.isnull()] = df_s_9_tavg.mean()
df_s_10_tavg_with_mean = df_s_10_tavg.copy(); df_s_10_tavg_with_mean[df_s_10_tavg_with_mean.isnull()] = df_s_10_tavg.mean()
df_s_11_tavg_with_mean = df_s_11_tavg.copy(); df_s_11_tavg_with_mean[df_s_11_tavg_with_mean.isnull()] = df_s_11_tavg.mean()
df_s_12_tavg_with_mean = df_s_12_tavg.copy(); df_s_12_tavg_with_mean[df_s_12_tavg_with_mean.isnull()] = df_s_12_tavg.mean()
df_s_13_tavg_with_mean = df_s_13_tavg.copy(); df_s_13_tavg_with_mean[df_s_13_tavg_with_mean.isnull()] = df_s_13_tavg.mean()
df_s_14_tavg_with_mean = df_s_14_tavg.copy(); df_s_14_tavg_with_mean[df_s_14_tavg_with_mean.isnull()] = df_s_14_tavg.mean()
df_s_15_tavg_with_mean = df_s_15_tavg.copy(); df_s_15_tavg_with_mean[df_s_15_tavg_with_mean.isnull()] = df_s_15_tavg.mean()
df_s_16_tavg_with_mean = df_s_16_tavg.copy(); df_s_16_tavg_with_mean[df_s_16_tavg_with_mean.isnull()] = df_s_16_tavg.mean()
df_s_17_tavg_with_mean = df_s_17_tavg.copy(); df_s_17_tavg_with_mean[df_s_17_tavg_with_mean.isnull()] = df_s_17_tavg.mean()
df_s_18_tavg_with_mean = df_s_18_tavg.copy(); df_s_18_tavg_with_mean[df_s_18_tavg_with_mean.isnull()] = df_s_18_tavg.mean()
df_s_19_tavg_with_mean = df_s_19_tavg.copy(); df_s_19_tavg_with_mean[df_s_19_tavg_with_mean.isnull()] = df_s_19_tavg.mean()
df_s_20_tavg_with_mean = df_s_20_tavg.copy(); df_s_20_tavg_with_mean[df_s_20_tavg_with_mean.isnull()] = df_s_20_tavg.mean()
# -

# 각 각의 Station 별로 tavg의 값이 np.nan일 때, 즉, missing value를 뺏을 때의 전체 mean 값을 나타낸다. 
print('#station1_without_nan_mean:', round(df_s_1_tavg.mean(),4), '#station1_without_nan_std:', round(df_s_1_tavg.std(),4)); 
print('#station2_without_nan_mean:', round(df_s_2_tavg.mean(),4), '#station2_without_nan_std:', round(df_s_2_tavg.std(),4)); 
print('#station3_without_nan_mean:', round(df_s_3_tavg.mean(),4), '#station3_without_nan_std:', round(df_s_3_tavg.std(),4)); 
print('#station4_without_nan_mean:', round(df_s_4_tavg.mean(),4), '#station4_without_nan_std:', round(df_s_4_tavg.std(),4)); 
print('#station5_without_nan_mean:', round(df_s_5_tavg.mean(),4), '#station5_without_nan_std:', round(df_s_5_tavg.std(),4)); 
print('#station6_without_nan_mean:', round(df_s_6_tavg.mean(),4), '#station6_without_nan_std:', round(df_s_6_tavg.std(),4)); 
print('#station7_without_nan_mean:', round(df_s_7_tavg.mean(),4), '#station7_without_nan_std:', round(df_s_7_tavg.std(),4)); 
print('#station8_without_nan_mean:', round(df_s_8_tavg.mean(),4), '#station8_without_nan_std:', round(df_s_8_tavg.std(),4)); 
print('#station9_without_nan_mean:', round(df_s_9_tavg.mean(),4), '#station9_without_nan_std:', round(df_s_9_tavg.std(),4)); 
print('#station10_without_nan_mean:', round(df_s_10_tavg.mean(),4), '#station10_without_nan_std:', round(df_s_10_tavg.std(),4));  
print('#station11_without_nan_mean:', round(df_s_11_tavg.mean(),4), '#station11_without_nan_std:', round(df_s_11_tavg.std(),4)); 
print('#station12_without_nan_mean:', round(df_s_12_tavg.mean(),4), '#station12_without_nan_std:', round(df_s_12_tavg.std(),4)); 
print('#station13_without_nan_mean:', round(df_s_13_tavg.mean(),4), '#station13_without_nan_std:', round(df_s_13_tavg.std(),4)); 
print('#station14_without_nan_mean:', round(df_s_14_tavg.mean(),4), '#station14_without_nan_std:', round(df_s_14_tavg.std(),4)); 
print('#station15_without_nan_mean:', round(df_s_15_tavg.mean(),4), '#station15_without_nan_std:', round(df_s_15_tavg.std(),4)); 
print('#station16_without_nan_mean:', round(df_s_16_tavg.mean(),4), '#station16_without_nan_std:', round(df_s_16_tavg.std(),4)); 
print('#station17_without_nan_mean:', round(df_s_17_tavg.mean(),4), '#station17_without_nan_std:', round(df_s_17_tavg.std(),4)); 
print('#station18_without_nan_mean:', round(df_s_18_tavg.mean(),4), '#station18_without_nan_std:', round(df_s_18_tavg.std(),4)); 
print('#station19_without_nan_mean:', round(df_s_19_tavg.mean(),4), '#station19_without_nan_std:', round(df_s_19_tavg.std(),4)); 
print('#station20_without_nan_mean:', round(df_s_20_tavg.mean(),4), '#station20_without_nan_std:', round(df_s_20_tavg.std(),4)); 

print('stat1_nan_as_mean:',round(df_s_1_tavg_with_mean.mean(),4),'#stat1_nan_as_std:', round(df_s_1_tavg_with_mean.std(),4));
print('stat2_nan_as_mean:',round(df_s_2_tavg_with_mean.mean(),4),'#stat2_nan_as_std:', round(df_s_2_tavg_with_mean.std(),4));
print('stat3_nan_as_mean:',round(df_s_3_tavg_with_mean.mean(),4),'#stat3_nan_as_std:', round(df_s_3_tavg_with_mean.std(),4));
print('stat4_nan_as_mean:',round(df_s_4_tavg_with_mean.mean(),4),'#stat4_nan_as_std:', round(df_s_4_tavg_with_mean.std(),4));
print('stat5_nan_as_mean:',round(df_s_5_tavg_with_mean.mean(),4),'#stat5_nan_as_std:', round(df_s_5_tavg_with_mean.std(),4));
print('stat6_nan_as_mean:',round(df_s_6_tavg_with_mean.mean(),4),'#stat6_nan_as_std:', round(df_s_6_tavg_with_mean.std(),4));
print('stat7_nan_as_mean:',round(df_s_7_tavg_with_mean.mean(),4),'#stat7_nan_as_std:', round(df_s_7_tavg_with_mean.std(),4));
print('stat8_nan_as_mean:',round(df_s_8_tavg_with_mean.mean(),4),'#stat8_nan_as_std:', round(df_s_8_tavg_with_mean.std(),4));
print('stat9_nan_as_mean:',round(df_s_9_tavg_with_mean.mean(),4),'#stat9_nan_as_std:', round(df_s_9_tavg_with_mean.std(),4));
print('stat10_nan_as_mean:',round(df_s_10_tavg_with_mean.mean(),4),'#stat10_nan_as_std:', round(df_s_10_tavg_with_mean.std(),4));
print('stat11_nan_as_mean:',round(df_s_11_tavg_with_mean.mean(),4),'#stat11_nan_as_std:', round(df_s_11_tavg_with_mean.std(),4));
print('stat12_nan_as_mean:',round(df_s_12_tavg_with_mean.mean(),4),'#stat12_nan_as_std:', round(df_s_12_tavg_with_mean.std(),4));
print('stat13_nan_as_mean:',round(df_s_13_tavg_with_mean.mean(),4),'#stat13_nan_as_std:', round(df_s_13_tavg_with_mean.std(),4));
print('stat14_nan_as_mean:',round(df_s_14_tavg_with_mean.mean(),4),'#stat14_nan_as_std:', round(df_s_14_tavg_with_mean.std(),4));
print('stat15_nan_as_mean:',round(df_s_15_tavg_with_mean.mean(),4),'#stat15_nan_as_std:', round(df_s_15_tavg_with_mean.std(),4));
print('stat16_nan_as_mean:',round(df_s_16_tavg_with_mean.mean(),4),'#stat16_nan_as_std:', round(df_s_16_tavg_with_mean.std(),4));
print('stat17_nan_as_mean:',round(df_s_17_tavg_with_mean.mean(),4),'#stat17_nan_as_std:', round(df_s_17_tavg_with_mean.std(),4));
print('stat18_nan_as_mean:',round(df_s_18_tavg_with_mean.mean(),4),'#stat18_nan_as_std:', round(df_s_18_tavg_with_mean.std(),4));
print('stat19_nan_as_mean:',round(df_s_19_tavg_with_mean.mean(),4),'#stat19_nan_as_std:', round(df_s_19_tavg_with_mean.std(),4));
print('stat20_nan_as_mean:',round(df_s_20_tavg_with_mean.mean(),4),'#stat20_nan_as_std:', round(df_s_20_tavg_with_mean.std(),4));

# +
y1 = np.array([df_s_1_tavg.mean(), df_s_2_tavg.mean(), df_s_3_tavg.mean(), df_s_4_tavg.mean(), df_s_5_tavg.mean(),
             df_s_6_tavg.mean(), df_s_7_tavg.mean(), df_s_8_tavg.mean(), df_s_9_tavg.mean(), df_s_10_tavg.mean(), 
             df_s_11_tavg.mean(), df_s_12_tavg.mean(), df_s_13_tavg.mean(), df_s_14_tavg.mean(), df_s_15_tavg.mean(),
             df_s_16_tavg.mean(), df_s_17_tavg.mean(), df_s_18_tavg.mean(), df_s_19_tavg.mean(), df_s_20_tavg.mean()])

y2 = np.array([df_s_1_tavg_with_mean.mean(), df_s_2_tavg_with_mean.mean(), df_s_3_tavg_with_mean.mean(), df_s_4_tavg_with_mean.mean()
             ,df_s_5_tavg_with_mean.mean(), df_s_6_tavg_with_mean.mean(), df_s_7_tavg_with_mean.mean(), df_s_8_tavg_with_mean.mean()
             ,df_s_9_tavg_with_mean.mean(), df_s_10_tavg_with_mean.mean(), df_s_11_tavg_with_mean.mean(), df_s_12_tavg_with_mean.mean()
             ,df_s_13_tavg_with_mean.mean(), df_s_14_tavg_with_mean.mean(), df_s_15_tavg_with_mean.mean(), df_s_16_tavg_with_mean.mean()
             ,df_s_17_tavg_with_mean.mean(),df_s_18_tavg_with_mean.mean(),df_s_19_tavg_with_mean.mean(),df_s_20_tavg_with_mean.mean()])

plt.figure(figsize=(10,5))
plt.plot()
x = range(1, 21)
plt.xlabel('Station Number')
plt.ylabel('tavg')
plt.bar(x, y1, color='y')
plt.bar(x, y2, color='r')
plt.show()

#### tavg 에서는
# 이 Graph가 나타내는 바는, y1과 y2가 차이가 거의 없다는 것. 그래프에는 r color와 y color이 있는데, 하나만 보인다는 것은
# 우리가 그래프에서 눈으로 보이지 않는 정도의 차이가 나온다는 것이다, 즉 차이가 극히 작다고 판단 할 수 있다.
# Missing Values를 np.nan으로 바꾸고, 거기서 Mean 값을 구하여 np.nan에 대입하여도, 
# 따라서, 여기서는 Missing Value에 Mean값을 적용하여 회귀 분석에 적용 할 수있다고 판단된다. 
# -

plt.figure(figsize=(12,6))
xticks = range(1,21)
plt.plot(xticks, y1, 'ro-', label='stations_without_nan_mean')
plt.plot(xticks, y2, 'bd:', label='stations_nan_as_mean')
plt.legend(loc=0)
plt.show()

# +
y3 = np.array([df_s_1_tavg.std(), df_s_2_tavg.std(), df_s_3_tavg.std(), df_s_4_tavg.std(), df_s_5_tavg.std(),
             df_s_6_tavg.std(), df_s_7_tavg.std(), df_s_8_tavg.std(), df_s_9_tavg.std(), df_s_10_tavg.std(), 
             df_s_11_tavg.std(), df_s_12_tavg.std(), df_s_13_tavg.std(), df_s_14_tavg.std(), df_s_15_tavg.std(),
             df_s_16_tavg.std(), df_s_17_tavg.std(), df_s_18_tavg.std(), df_s_19_tavg.std(), df_s_20_tavg.std()])

y4 = np.array([df_s_1_tavg_with_mean.std(), df_s_2_tavg_with_mean.std(), df_s_3_tavg_with_mean.std(), df_s_4_tavg_with_mean.std()
             ,df_s_5_tavg_with_mean.std(), df_s_6_tavg_with_mean.std(), df_s_7_tavg_with_mean.std(), df_s_8_tavg_with_mean.std()
             ,df_s_9_tavg_with_mean.std(), df_s_10_tavg_with_mean.std(), df_s_11_tavg_with_mean.std(), df_s_12_tavg_with_mean.std()
             ,df_s_13_tavg_with_mean.std(), df_s_14_tavg_with_mean.std(), df_s_15_tavg_with_mean.std(), df_s_16_tavg_with_mean.std()
             ,df_s_17_tavg_with_mean.std(),df_s_18_tavg_with_mean.std(),df_s_19_tavg_with_mean.std(),df_s_20_tavg_with_mean.std()])

plt.figure(figsize=(10,5))
plt.plot()
x = range(1, 21)
plt.xlabel('Station Number')
plt.ylabel('tavg')
plt.bar(x, y3, color='r')
plt.bar(x, y4, color='g')
plt.show()
# -

plt.figure(figsize=(12,6))
xticks = range(1,21)
plt.plot(xticks, y3, 'ro-', label='stations_without_nan_std')
plt.plot(xticks, y4, 'bd:', label='stations_nan_as_std')
plt.legend(loc=0)
plt.show()



StarcoderdataJupyterScriptsDedupFiltered