"""
Machine Learning vs. Deep Learning
"""
# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import matplotlib.pyplot as plt
import seaborn as sns

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
resp_2021 = pd.read_csv('/kaggle/input/kaggle-survey-2021/kaggle_survey_2021_responses.csv')
resp_2020 = pd.read_csv('/kaggle/input/kaggle-survey-2020/kaggle_survey_2020_responses.csv')
resp_2019 = pd.read_csv('/kaggle/input/kaggle-survey-2019/multiple_choice_responses.csv')
resp_2018 = pd.read_csv('/kaggle/input/kaggle-survey-2018/multipleChoiceResponses.csv')
resp_2017 = pd.read_csv('/kaggle/input/kaggle-survey-2017/multipleChoiceResponses.csv', encoding='ISO-8859-1')
schema_2017 = pd.read_csv('/kaggle/input/kaggle-survey-2017/schema.csv', encoding='ISO-8859-1')
schema_2017
schema_2017[schema_2017['Question'].str.contains('hardware')]
gpu_res = dict()
gpu_res['Years'] = []
gpu_res['Accelerator Usage By Users'] = []
searchfor = ['GPU', 'TPU']

gpu_2017 = resp_2017[resp_2017['WorkHardwareSelect'].str.contains('|'.join(searchfor)).fillna(value=False)].shape[0] # gpu at work users in 2017 
gpu_total_2017 = sum(resp_2017['WorkHardwareSelect'].str.contains('GPU').value_counts()) #total number of people answering question at work


gpu_2017 += resp_2017[resp_2017['HardwarePersonalProjectsSelect'].str.contains('|'.join(searchfor)).fillna(value=False)].shape[0] #added personal project gpu numbers
gpu_total_2017 += sum(resp_2017['HardwarePersonalProjectsSelect'].str.contains('GPU').value_counts())

gpu_res['Years'].append('2017')
gpu_res['Accelerator Usage By Users'].append(gpu_2017 / gpu_total_2017)
#2018년에 gpu data 존재 X 
schema_2019 = pd.read_csv('/kaggle/input/kaggle-survey-2019/questions_only.csv', encoding='ISO-8859-1').transpose() #돌려야지 데이터가 보기 쉽게 나옴 궁금하면 transpose 없이 해보면 됨
schema_2019[schema_2019[0].str.contains('hardware')] #Q21
#같은 사람을 두번 세지 않게 주의 !! 

gpu_total_2019 = resp_2019['Q21_Part_1'].str.contains('CPUs').fillna(value = False) | \
resp_2019['Q21_Part_2'].str.contains('GPUs').fillna(value = False) | \
resp_2019['Q21_Part_3'].str.contains('TPUs').fillna(value = False) | \
resp_2019['Q21_Part_4'].str.contains('None').fillna(value = False) | \
resp_2019['Q21_Part_5'].str.contains('Other').fillna(value = False) #boolean indexing to get unique users

gpu_total_2019 = gpu_total_2019.value_counts().to_frame().transpose()[True][0] - 5 #minus to account for question itself

gpu_2019 = resp_2019['Q21_Part_2'].str.contains('GPUs').fillna(value = False) | \
resp_2019['Q21_Part_3'].str.contains('TPUs').fillna(value = False)

gpu_2019 = gpu_2019.value_counts().to_frame().transpose()[True][0] - 2 #minus to account for question itself


gpu_res['Years'].append('2019')
gpu_res['Accelerator Usage By Users'].append(gpu_2019 / gpu_total_2019)
#2020 ==> Q12

gpu_total_2020 = resp_2020['Q12_Part_1'].str.contains('GPUs').fillna(value = False) | \
resp_2020['Q12_Part_2'].str.contains('TPUs').fillna(value = False) | \
resp_2020['Q12_Part_3'].str.contains('None').fillna(value = False) | \
resp_2020['Q12_OTHER'].str.contains('Other').fillna(value = False) 

gpu_total_2020 = gpu_total_2020.value_counts().to_frame().transpose()[True][0] - 4 #minus to account for question itself

gpu_2020 = resp_2020['Q12_Part_1'].str.contains('GPUs').fillna(value = False) | \
resp_2020['Q12_Part_2'].str.contains('TPUs').fillna(value = False)

gpu_2020 = gpu_2020.value_counts().to_frame().transpose()[True][0] - 2 #minus to account for question itself

gpu_res['Years'].append('2020')
gpu_res['Accelerator Usage By Users'].append(gpu_2020 / gpu_total_2020)
#2021 --> Q12

gpu_total_2021 = resp_2021['Q12_Part_1'].str.contains('GPUs').fillna(value = False) | \
resp_2021['Q12_Part_2'].str.contains('TPUs').fillna(value = False) | \
resp_2021['Q12_Part_3'].str.contains('AWS Trainium Chips').fillna(value = False) | \
resp_2021['Q12_Part_4'].str.contains('AWS Inferentia Chips').fillna(value = False) | \
resp_2021['Q12_Part_5'].str.contains('None').fillna(value = False) | \
resp_2021['Q12_OTHER'].str.contains('Other').fillna(value = False)

gpu_total_2021 = gpu_total_2021.value_counts().to_frame().transpose()[True][0] - 6 #minus to account for question itself

gpu_2021 = resp_2021['Q12_Part_1'].str.contains('GPUs').fillna(value = False) | \
resp_2021['Q12_Part_2'].str.contains('TPUs').fillna(value = False) | \
resp_2021['Q12_Part_3'].str.contains('AWS Trainium Chips').fillna(value = False) | \
resp_2021['Q12_Part_4'].str.contains('AWS Inferentia Chips').fillna(value = False)

#gpu_2021.value_counts()

gpu_2021 = gpu_2021.value_counts().to_frame().transpose()[True][0] - 4 #minus to account for question itself

gpu_res['Years'].append('2021')
gpu_res['Accelerator Usage By Users'].append(gpu_2021 / gpu_total_2021)
gpu_res['Accelerator Usage By Users']
gpu_res['Style'] = ['Accelerator Usage By Users' for _ in range(4)]
sns.lineplot(data=gpu_res, x='Years', y='Accelerator Usage By Users', style = 'Style', markers=True)
#plt.plot(gpu_res['Accelerator Usage By Users'])
#Which ML algos do you use on a regular basis

#2021 Q17
#2020 Q17
#2019 Q24
#2018 none
#2017 WorkAlgorithmsSelect
#interesting datapoint in 2017 --> chooseOne("MLMethodNextYearSelect")
#shows that 40% of users indicated they were interested in deep learning and 12% in Neural Nets
#so did they learn those?

#if there is plateau, does it mean that deep learning is difficult to use and may be too expensive and requires
#too much data so they just use machine learning techniques when they can, because performance is extremely
#important in real world applications, not just accuracy 
#like whats the point of using nn if xgboost does about the same thing but faster 
#also it may be that kaggle users usually find their niche in one area or another and just stick to that
#particular niche and specialize in it, so they'd rather just get better at let's say image processing
#and not attempt to try out RNNs or Transformers 
#maybe some techniques are just still in research stage and not easily applicable to real world yet
#so as deep nns advance in multiple fronts they will become more and more popular
responses_df_2021 = resp_2021

def count_then_return_percent_for_multiple_column_questions(dataframe,list_of_columns_for_a_single_question,dictionary_of_counts_for_a_single_question):
    '''
    A helper function to convert counts to percentages.
    '''
    df = dataframe
    subset = list_of_columns_for_a_single_question
    df = df[subset]
    df = df.dropna(how='all')
    total_count = len(df) 
    dictionary = dictionary_of_counts_for_a_single_question
    for i in dictionary:
        dictionary[i] = round(float(dictionary[i]*100/total_count),1)
    return dictionary


q17_list_of_columns_2021 = ['Q17_Part_1',
                       'Q17_Part_2',
                       'Q17_Part_3',
                       'Q17_Part_4',
                       'Q17_Part_5',
                       'Q17_Part_6',
                       'Q17_Part_7',
                       'Q17_Part_8',
                       'Q17_Part_9',
                       'Q17_Part_10',
                       'Q17_Part_11',
                       'Q17_OTHER']

q17_dictionary_of_counts_2021 = {
    'Linear or Logistic Regression' : (responses_df_2021['Q17_Part_1'].count()),
    'Decision Trees or Random Forests': (responses_df_2021['Q17_Part_2'].count()),
    'Gradient Boosting Machines (xgboost, lightgbm, etc)' : (responses_df_2021['Q17_Part_3'].count()),
    'Dense Neural Networks (MLPs, etc)' : (responses_df_2021['Q17_Part_6'].count()),
    'Convolutional Neural Networks' : (responses_df_2021['Q17_Part_7'].count()),
    'Recurrent Neural Networks' : (responses_df_2021['Q17_Part_9'].count()),
    'Transformer Networks (BERT, gpt-3, etc)' : (responses_df_2021['Q17_Part_10'].count()),
}



q17_dictionary_of_perc_2021 = count_then_return_percent_for_multiple_column_questions(responses_df_2021,
                                                  q17_list_of_columns_2021,
                                                  q17_dictionary_of_counts_2021)

responses_df_2020 = resp_2020

q17_list_of_columns_2020 = ['Q17_Part_1',
                       'Q17_Part_2',
                       'Q17_Part_3',
                       'Q17_Part_4',
                       'Q17_Part_5',
                       'Q17_Part_6',
                       'Q17_Part_7',
                       'Q17_Part_8',
                       'Q17_Part_9',
                       'Q17_Part_10',
                       'Q17_Part_11',
                       'Q17_OTHER']

q17_dictionary_of_counts_2020 = {
    'Linear or Logistic Regression' : (responses_df_2020['Q17_Part_1'].count()),
    'Decision Trees or Random Forests': (responses_df_2020['Q17_Part_2'].count()),
    'Gradient Boosting Machines (xgboost, lightgbm, etc)' : (responses_df_2020['Q17_Part_3'].count()),
    'Dense Neural Networks (MLPs, etc)' : (responses_df_2020['Q17_Part_6'].count()),
    'Convolutional Neural Networks' : (responses_df_2020['Q17_Part_7'].count()),
    'Recurrent Neural Networks' : (responses_df_2020['Q17_Part_9'].count()),
    'Transformer Networks (BERT, gpt-3, etc)' : (responses_df_2020['Q17_Part_10'].count()),
}

q17_dictionary_of_perc_2020 = count_then_return_percent_for_multiple_column_questions(responses_df_2020,
                                                  q17_list_of_columns_2020,
                                                  q17_dictionary_of_counts_2020)
#q24 2019

q24_list_of_columns_2019 = ['Q24_Part_1',
                       'Q24_Part_2',
                       'Q24_Part_3',
                       'Q24_Part_4',
                       'Q24_Part_5',
                       'Q24_Part_6',
                       'Q24_Part_7',
                       'Q24_Part_8',
                       'Q24_Part_9',
                       'Q24_Part_10',
                       'Q24_Part_11',
                       'Q24_Part_12']

q24_dictionary_of_counts_2019 = {
    'Linear or Logistic Regression' : (resp_2019['Q24_Part_1'].count()),
    'Decision Trees or Random Forests': (resp_2019['Q24_Part_2'].count()),
    'Gradient Boosting Machines (xgboost, lightgbm, etc)' : (resp_2019['Q24_Part_3'].count()),
    'Dense Neural Networks (MLPs, etc)' : (resp_2019['Q24_Part_6'].count()),
    'Convolutional Neural Networks' : (resp_2019['Q24_Part_7'].count()),
    'Recurrent Neural Networks' : (resp_2019['Q24_Part_9'].count()),
    'Transformer Networks (BERT, gpt-3, etc)' : (resp_2019['Q24_Part_10'].count()),
}

q24_dictionary_of_perc_2019 = count_then_return_percent_for_multiple_column_questions(resp_2019,
                                                  q24_list_of_columns_2019,
                                                  q24_dictionary_of_counts_2019)
#2017

searchfor = ['Decision Trees', 'Random Forests']

def count_2017_q_workalg(dictionary_of_counts_for_a_single_question):
    '''
    A helper function to convert counts to percentages.
    '''
    total_count = 7301 #number taken from 2017 kaggle survey analysis kernel by the pudding
    dictionary = dictionary_of_counts_for_a_single_question
    for i in dictionary:
        dictionary[i] = round(float(dictionary[i]*100/total_count),1)
    return dictionary

q_workalg_dictionary_of_counts_2017 = { 

    'Linear or Logistic Regression' : resp_2017['WorkAlgorithmsSelect'].str.contains('Regression/Logistic Regression').value_counts().to_frame().transpose()[True][0],
    'Decision Trees or Random Forests': resp_2017['WorkAlgorithmsSelect'].str.contains('|'.join(searchfor)).value_counts().to_frame().transpose()[True][0],
    'Gradient Boosting Machines (xgboost, lightgbm, etc)' : resp_2017['WorkAlgorithmsSelect'].str.contains('Gradient Boosted Machines').value_counts().to_frame().transpose()[True][0],
    'Dense Neural Networks (MLPs, etc)' : resp_2017['WorkAlgorithmsSelect'].str.contains('Neural Networks').value_counts().to_frame().transpose()[True][0],
    'Convolutional Neural Networks' : resp_2017['WorkAlgorithmsSelect'].str.contains('CNNs').value_counts().to_frame().transpose()[True][0],
    'Recurrent Neural Networks' : resp_2017['WorkAlgorithmsSelect'].str.contains('RNNs').value_counts().to_frame().transpose()[True][0],
    'Transformer Networks (BERT, gpt-3, etc)' : 0

}

q24_dictionary_of_perc_2017 = count_2017_q_workalg(q_workalg_dictionary_of_counts_2017)
q17_dictionary_of_perc_2021
#scatterplot work

def algo_type_scatterplot(perc_dict, year):
    df = pd.DataFrame(data = perc_dict, index = ['Percentage of Respondents']).transpose()
    df['Year'] = year
    df = df.reset_index()
    df = df.rename(columns={'index':'Type of Algorithm'})
    
    return df
    

algo_type_df = algo_type_scatterplot(q17_dictionary_of_perc_2021, 2021)
algo_type_df = algo_type_df.append(algo_type_scatterplot(q17_dictionary_of_perc_2020, 2020))
algo_type_df = algo_type_df.append(algo_type_scatterplot(q24_dictionary_of_perc_2019, 2019))
algo_type_df = algo_type_df.append(algo_type_scatterplot(q24_dictionary_of_perc_2017, 2017))
algo_type_df = algo_type_df.reset_index(drop=True)
a4_dims = (13, 10)
algo_fig, algo_ax = plt.subplots(figsize=a4_dims)

sns.lineplot(data = algo_type_df,
            ax = algo_ax,
            x = 'Year',
            y = 'Percentage of Respondents',
            hue = 'Type of Algorithm',
            style = 'Type of Algorithm',
            markers = True,
            dashes = False)
algo_ax.set_xticks(ticks = [2017, 2019, 2020, 2021])
#cloud compute
#popularity of cloud compute show that users who don't use any are decreasing
#show that trend in money used for cloud is not changing much maybe since cloud is expensive
#credits to kaggle's eda notebook

responses_df_2018 = resp_2018
responses_df_2019 = resp_2019

q26a_dictionary_of_counts_2018 = {
    'Amazon Web Services (AWS)' : (responses_df_2018['Q15_Part_2'].count()),
    'Microsoft Azure': (responses_df_2018['Q15_Part_3'].count()),
    'Google Cloud Platform (GCP)' : (responses_df_2018['Q15_Part_1'].count()),
    'None' : (responses_df_2018['Q15_Part_6'].count()),
}

q26a_list_of_columns_2018 = ['Q15_Part_1',
                        'Q15_Part_2',
                        'Q15_Part_3',
                        'Q15_Part_4',
                        'Q15_Part_5',
                        'Q15_Part_6',
                        'Q15_Part_7']

q26a_dictionary_of_counts_2019 = {
    'Amazon Web Services (AWS)' : (responses_df_2019['Q29_Part_2'].count()),
    'Microsoft Azure': (responses_df_2019['Q29_Part_3'].count()),
    'Google Cloud Platform (GCP)' : (responses_df_2019['Q29_Part_1'].count()),
    'None' : (responses_df_2019['Q29_Part_11'].count()),
}

q26a_list_of_columns_2019 = ['Q29_Part_1',
                        'Q29_Part_2',
                        'Q29_Part_3',
                        'Q29_Part_4',
                        'Q29_Part_5',
                        'Q29_Part_6',
                        'Q29_Part_7',
                        'Q29_Part_8',
                        'Q29_Part_9',
                        'Q29_Part_10',
                        'Q29_Part_11',
                        'Q29_Part_12']

q26a_dictionary_of_counts_2020 = {
    'Amazon Web Services (AWS)' : (responses_df_2020['Q26_A_Part_1'].count()),
    'Microsoft Azure': (responses_df_2020['Q26_A_Part_2'].count()),
    'Google Cloud Platform (GCP)' : (responses_df_2020['Q26_A_Part_3'].count()),
    'None' : (responses_df_2020['Q26_A_Part_11'].count()),
}

q26a_list_of_columns_2020 = ['Q26_A_Part_1',
                        'Q26_A_Part_2',
                        'Q26_A_Part_3',
                        'Q26_A_Part_4',
                        'Q26_A_Part_5',
                        'Q26_A_Part_6',
                        'Q26_A_Part_7',
                        'Q26_A_Part_8',
                        'Q26_A_Part_9',
                        'Q26_A_Part_10',
                        'Q26_A_Part_11',
                        'Q26_A_OTHER']

q27a_dictionary_of_counts_2021 = {
    'Amazon Web Services (AWS)' : (responses_df_2021['Q27_A_Part_1'].count()),
    'Microsoft Azure': (responses_df_2021['Q27_A_Part_2'].count()),
    'Google Cloud Platform (GCP)' : (responses_df_2021['Q27_A_Part_3'].count()),
    'None' : (responses_df_2021['Q27_A_Part_11'].count()),
}

q27a_list_of_columns_2021 = ['Q27_A_Part_1',
                        'Q27_A_Part_2',
                        'Q27_A_Part_3',
                        'Q27_A_Part_4',
                        'Q27_A_Part_5',
                        'Q27_A_Part_6',
                        'Q27_A_Part_7',
                        'Q27_A_Part_8',
                        'Q27_A_Part_9',
                        'Q27_A_Part_10',
                        'Q27_A_Part_11',
                        'Q27_A_OTHER']

q26a_dictionary_of_perc_2018 = count_then_return_percent_for_multiple_column_questions(responses_df_2018,
                                                  q26a_list_of_columns_2018,
                                                  q26a_dictionary_of_counts_2018)
q26a_dictionary_of_perc_2019 = count_then_return_percent_for_multiple_column_questions(responses_df_2019,
                                                  q26a_list_of_columns_2019,
                                                  q26a_dictionary_of_counts_2019)
q26a_dictionary_of_perc_2020 = count_then_return_percent_for_multiple_column_questions(responses_df_2020,
                                                  q26a_list_of_columns_2020,
                                                  q26a_dictionary_of_counts_2020)
q27a_dictionary_of_perc_2021 = count_then_return_percent_for_multiple_column_questions(responses_df_2021,
                                                  q27a_list_of_columns_2021,
                                                  q27a_dictionary_of_counts_2021)

def cloud_scatterplot(perc_dict, year):
    df = pd.DataFrame(data = perc_dict, index = ['Percentage of Respondents']).transpose()
    df['Year'] = year
    df = df.reset_index()
    df = df.rename(columns={'index':'Cloud Computing Platform'})
    
    return df

cloud_use_df = cloud_scatterplot(q27a_dictionary_of_perc_2021, 2021)
cloud_use_df = cloud_use_df.append(cloud_scatterplot(q26a_dictionary_of_perc_2020, 2020))
cloud_use_df = cloud_use_df.append(cloud_scatterplot(q26a_dictionary_of_perc_2019, 2019))
cloud_use_df = cloud_use_df.append(cloud_scatterplot(q26a_dictionary_of_perc_2018, 2018))
cloud_use_df = cloud_use_df.reset_index(drop=True)
a4_dims = (13, 10)
cloud_fig, cloud_ax = plt.subplots(figsize=a4_dims)

sns.lineplot(data = cloud_use_df,
            ax = cloud_ax,
            x = 'Year',
            y = 'Percentage of Respondents',
            hue = 'Cloud Computing Platform',
            style = 'Cloud Computing Platform',
            markers = True,
            dashes = False)
cloud_ax.set_xticks(ticks = [2018, 2019, 2020, 2021])
def count_then_return_percent(dataframe,column_name):
    '''
    A helper function to return value counts as percentages.
    '''
    counts = dataframe[column_name].value_counts(dropna=True)
    percentages = round(counts*100/(dataframe[column_name].count()),1)
    return percentages

#responses_df_2019 = responses_df_2019['Q11'].replace(["$0 (USD)"], "$0",inplace=True)

def cloud_cost_scatterplot(perc_dict, year):
    df = perc_dict.to_frame(name = 'Percentage of Respondents').iloc[1:,:]
    df = df.reset_index()
    df = df.rename(columns={'index': 'Money Spent on Cloud(USD)'})
    df['Year'] = year
    
    return df

cloud_cost_perc_2019 = count_then_return_percent(responses_df_2019,'Q11').iloc[::-1]#[responses_in_order]
cloud_cost_perc_2020 = count_then_return_percent(responses_df_2020,'Q25').iloc[::-1]#[responses_in_order]
cloud_cost_perc_2021 = count_then_return_percent(responses_df_2021,'Q26').iloc[::-1]#[responses_in_order]

cloud_cost_df = cloud_cost_scatterplot(cloud_cost_perc_2021, 2021)
cloud_cost_df = cloud_cost_df.append(cloud_cost_scatterplot(cloud_cost_perc_2020, 2020))
cloud_cost_df = cloud_cost_df.append(cloud_cost_scatterplot(cloud_cost_perc_2019, 2019))
cloud_cost_df = cloud_cost_df.reset_index(drop=True)
a4_dims = (13, 10)
cloud_cost_fig, cloud_cost_ax = plt.subplots(figsize=a4_dims)

sns.lineplot(data = cloud_cost_df,
            ax = cloud_cost_ax,
            x = 'Year',
            y = 'Percentage of Respondents',
            hue = 'Money Spent on Cloud(USD)',
            style = 'Money Spent on Cloud(USD)',
            markers = True,
            dashes = False)
cloud_ax.set_xticks(ticks = [2019, 2020, 2021])

#shows that more people are using, but using free tier, perhaps to experiment
#유저 분석
gpu_total_2021 = resp_2021['Q17_Part_1'].str.contains('GPUs').fillna(value = False) | \
resp_2021['Q17_Part_2'].str.contains('TPUs').fillna(value = False) | \
resp_2021['Q12_Part_3'].str.contains('AWS Trainium Chips').fillna(value = False) | \
resp_2021['Q12_Part_4'].str.contains('AWS Inferentia Chips').fillna(value = False) | \
resp_2021['Q12_Part_5'].str.contains('None').fillna(value = False) | \
resp_2021['Q12_OTHER'].str.contains('Other').fillna(value = False)
(pd.Series(['hi', 'hello']).str.contains('')) & (pd.Series(['hey', 'ho']).str.contains('hey'))
resp_2021.iloc[8].to_frame().reset_index().iloc[90:102]
resp_2021.iloc[0:10]
#유저를 3으로 나눈다  --> 머신러닝 only  / 딥러닝 only / 머신러닝 + 딥러닝 
#년도마다 추이? 년도 + 머신러닝 pref + 다른 변수 분석
# --> 연봉?
# 직종???

'''Don't count other, none''' 

'''
Machine Learning Algos 
Lin Log Reg (1)
Decision (2)
Grad Boosting (3)
Bayesian (4)
Evolutionary (5)
'''

'''
Deep Learning Algos 
Dense NN (6)
CNN (7)
GAN (8)
RNN (9)
Transformer (10)
'''


ml_only_2021 = (resp_2021['Q17_Part_1'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_2'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_3'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_4'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_5'].str.contains('').fillna(value = False)) & \
(resp_2021['Q17_Part_6'].str.contains('impossible_val').fillna(value = True) & \
resp_2021['Q17_Part_7'].str.contains('impossible_val').fillna(value = True) & \
resp_2021['Q17_Part_8'].str.contains('impossible_val').fillna(value = True) & \
resp_2021['Q17_Part_9'].str.contains('impossible_val').fillna(value = True) & \
resp_2021['Q17_Part_10'].str.contains('impossible_val').fillna(value = True))

ml_only_2021 = resp_2021[ml_only_2021]
ml_only_2021 = ml_only_2021[(ml_only_2021['Q25'] != '$0-999') &  
                            (ml_only_2021['Q25'] != 'What is your current yearly compensation (approximate $USD)?')]

dl_only_2021 = (resp_2021['Q17_Part_6'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_7'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_8'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_9'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_10'].str.contains('').fillna(value = False)) & \
(resp_2021['Q17_Part_1'].str.contains('impossible_val').fillna(value = True) & \
resp_2021['Q17_Part_2'].str.contains('impossible_val').fillna(value = True) & \
resp_2021['Q17_Part_3'].str.contains('impossible_val').fillna(value = True) & \
resp_2021['Q17_Part_4'].str.contains('impossible_val').fillna(value = True) &  \
resp_2021['Q17_Part_5'].str.contains('impossible_val').fillna(value = True))
dl_only_2021 = resp_2021[dl_only_2021]
dl_only_2021 = dl_only_2021[(dl_only_2021['Q25'] != '$0-999') &  
                            (dl_only_2021['Q25'] != 'What is your current yearly compensation (approximate $USD)?')]

ml_dl_2021 = (resp_2021['Q17_Part_1'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_2'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_3'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_4'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_5'].str.contains('').fillna(value = False)) & \
(resp_2021['Q17_Part_6'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_7'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_8'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_9'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_10'].str.contains('').fillna(value = False))
ml_dl_2021 = resp_2021[ml_dl_2021]
ml_dl_2021 = ml_dl_2021[(ml_dl_2021['Q25'] != '$0-999') &  
                            (ml_dl_2021['Q25'] != 'What is your current yearly compensation (approximate $USD)?')]

mldl_count_check = (resp_2021['Q17_Part_1'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_2'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_3'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_4'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_5'].str.contains('').fillna(value = False)) | \
(resp_2021['Q17_Part_6'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_7'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_8'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_9'].str.contains('').fillna(value = False) | \
resp_2021['Q17_Part_10'].str.contains('').fillna(value = False))
mldl_count_check = resp_2021[mldl_count_check]
mldl_count_check = mldl_count_check[(mldl_count_check['Q25'] != '$0-999') &  
                            (mldl_count_check['Q25'] != 'What is your current yearly compensation (approximate $USD)?')]


def salary_conv_2021(df):
    #for responses in 2020
    #converts from strings to numerical data
    #I put inputted the value of salary that was right in between the range of salaries in order for a fair estimate of salaries
    conv_dict = {'10,000-14,999':12500, '1,000-1,999':1500, '100,000-124,999': 112500,
                '40,000-49,999':45000, '30,000-39,999':35000, '50,000-59,999': 55000, '5,000-7,499':6250,
                '15,000-19,999':17500, '60,000-69,999':65000, '20,000-24,999': 22500, '70,000-79,999':75000,
                '7,500-9,999':8750, '150,000-199,999':175000, '2,000-2,999':2500, '125,000-149,999':137500,
                '25,000-29,999':27500, '90,000-99,999':95000, '4,000-4,999':4500, '80,000-89,999':85000,
                '3,000-3,999':3500, '200,000-249,999':225000, '300,000-500,000':400000, '$500,000-999,999':500000,
                '250,000-299,999':275000, '>$1,000,000': 1000000, '300,000-499,999':350000}
    df = df['Q25'].map(conv_dict)
    return df
print(len(mldl_count_check)) 
print((len(ml_only_2021) + len(dl_only_2021)) + ( len(ml_dl_2021)))
#if same, the stuff is done correctly 
print(salary_conv_2021(ml_only_2021).mean())
print(salary_conv_2021(dl_only_2021).mean())
print(salary_conv_2021(ml_dl_2021).mean())
print(salary_conv_2021(mldl_count_check).mean())
print(salary_conv_2021(ml_only_2021).median())
print(salary_conv_2021(dl_only_2021).median())
print(salary_conv_2021(ml_dl_2021).median())
print(salary_conv_2021(mldl_count_check).median())
hi = salary_conv_2021(ml_only_2021).value_counts()
(salary_conv_2021(ml_only_2021).value_counts() / sum(hi)).to_frame().reset_index()
ml_only_2021
def df_to_salary_bar(series, x_ind, y_ind, specialty):
    val_counts = salary_conv_2021(series).value_counts()
    res = ((val_counts / sum(val_counts)) * 100).to_frame().reset_index()
    res = res.rename(columns = {'index':x_ind, 'Q25': y_ind})
    res['Type'] = specialty
    return res
    
a4_dims = (13, 10)
hi_fig, hi_ax = plt.subplots(figsize=a4_dims)


#sns.histplot(data = salary_conv_2021(ml_only_2021))
sns.barplot(data = df_to_salary_bar(ml_only_2021, 'Salary (USD)', 'Percentage of Respondents', 'ML Only'), x = 'Salary (USD)', y = 'Percentage of Respondents')

hi_ax.tick_params(labelrotation=45)

a4_dims = (13, 10)
hi_fig, hi_ax = plt.subplots(figsize=a4_dims)

#sns.histplot(data = salary_conv_2021(dl_only_2021))
sns.barplot(data = df_to_salary_bar(dl_only_2021, 'Salary (USD)', 'Percentage of Respondents', 'DL Only'),  x = 'Salary (USD)', y = 'Percentage of Respondents')

hi_ax.tick_params(labelrotation=45)
a4_dims = (13, 10)
hi_fig, hi_ax = plt.subplots(figsize=a4_dims)

sns.barplot(data = df_to_salary_bar(ml_dl_2021, 'Salary (USD)', 'Percentage of Respondents', 'ML and DL'),  x = 'Salary (USD)', y = 'Percentage of Respondents')
#sns.histplot(data = salary_conv_2021(ml_dl_2021))
hi_ax.tick_params(labelrotation=45)
hi = df_to_salary_bar(ml_only_2021, 'Salary (USD)', 'Percentage of Respondents', 'ML Only')
hi = hi.append(df_to_salary_bar(dl_only_2021, 'Salary (USD)', 'Percentage of Respondents', 'DL Only'))
hi = hi.append(df_to_salary_bar(ml_dl_2021, 'Salary (USD)', 'Percentage of Respondents', 'ML and DL'))
hi
a4_dims = (13, 10)
hi_fig, hi_ax = plt.subplots(figsize=a4_dims)

#sns.histplot(data = hi, x='Salary (USD)', y = 'Percentage of Respondents', hue='Type', element = 'step')
a4_dims = (13, 10)
hi_fig, hi_ax = plt.subplots(figsize=a4_dims)
sns.boxplot(x=salary_conv_2021(ml_only_2021))
a4_dims = (13, 10)
hi_fig, hi_ax = plt.subplots(figsize=a4_dims)
sns.boxplot(x=salary_conv_2021(dl_only_2021))
a4_dims = (13, 10)
hi_fig, hi_ax = plt.subplots(figsize=a4_dims)
sns.boxplot(x=salary_conv_2021(ml_dl_2021))
hi
salary_conv_2021(ml_only_2021)
def ahh(df, name):
    res = salary_conv_2021(df).dropna().to_frame()
    res['Type'] = name
    
    return res
hey = ahh(ml_only_2021, 'ML Only')
hey = hey.append(ahh(dl_only_2021, 'DL Only'))
hey = hey.append(ahh(ml_dl_2021, 'ML and DL'))
hey = hey.reset_index(drop = True)
hey
a4_dims = (13, 10)
hi_fig, hi_ax = plt.subplots(figsize=a4_dims)
sns.violinplot(data = hey, y = 'Type', x = 'Q25', jitter = 0.55)
a4_dims = (13, 10)
hi_fig, hi_ax = plt.subplots(figsize=a4_dims)


sns.barplot(data = df_to_salary_bar(ml_only_2021, 'Salary (USD)', 'Percentage of Respondents', 'ML Only'), x = 'Salary (USD)', y = 'Percentage of Respondents', 
            color = 'blue', alpha = 1)
sns.barplot(data = df_to_salary_bar(dl_only_2021, 'Salary (USD)', 'Percentage of Respondents', 'DL Only'),  x = 'Salary (USD)', y = 'Percentage of Respondents', 
            color = 'green', alpha = 1)
sns.barplot(data = df_to_salary_bar(ml_dl_2021, 'Salary (USD)', 'Percentage of Respondents', 'ML and DL'),  x = 'Salary (USD)', y = 'Percentage of Respondents', 
            color = 'orange', alpha = 1)


hi_ax.tick_params(labelrotation=45)
hello = hi.pivot(index='Salary (USD)', columns ='Type', values = 'Percentage of Respondents')

hello.plot.bar(stacked = True, figsize = (13, 10))
salary_conv_2021(ml_only_2021)
q17_dictionary_of_counts_2021 = {
    'Linear or Logistic Regression' : (responses_df_2021['Q17_Part_1'].count()),
    'Decision Trees or Random Forests': (responses_df_2021['Q17_Part_2'].count()),
    'Gradient Boosting Machines (xgboost, lightgbm, etc)' : (responses_df_2021['Q17_Part_3'].count()),
    'Bayesian Approaches' : (responses_df_2021['Q17_Part_4'].count()),
    'Evolutionary Approaches' : (responses_df_2021['Q17_Part_5'].count()),
    'Dense Neural Networks (MLPs, etc)' : (responses_df_2021['Q17_Part_6'].count()),
    'Convolutional Neural Networks' : (responses_df_2021['Q17_Part_7'].count()),
    'Generative Adversarial Networks' : (responses_df_2021['Q17_Part_8'].count()),
    'Recurrent Neural Networks' : (responses_df_2021['Q17_Part_9'].count()),
    'Transformer Networks (BERT, gpt-3, etc)' : (responses_df_2021['Q17_Part_10'].count()),
    'None' : (responses_df_2021['Q17_Part_11'].count()),
    'Other' : (responses_df_2021['Q17_OTHER'].count())
}



q17_dictionary_of_perc_2021 = count_then_return_percent_for_multiple_column_questions(responses_df_2021,
                                                  q17_list_of_columns_2021,
                                                  q17_dictionary_of_counts_2021)
'''
업계는 너무 많아서 일단 보류
'''

#slope graph를 보여주고 제일 첫번째, 마지막 빼고는 들러리 식으로 만들기 
#이거를 클라우드가 아닌 ml / dl 퍼센테이지로? 


#업계마다 딥러닝 / 머신러닝 현황 얼마나 쓰이는지 그리고 업계 마다 더 쓰는 업계가 있는지 2018에 비해서 얼마나 바뀌었는지 

#업계마다 짤라서 클라우드에 쓰는 돈을 비교 

resp_2021['Q20'].value_counts()
hi = resp_2021[resp_2021['Q20'] == 'Academics/Education']['Q26'].value_counts()
hi / sum(hi)