# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# **Business Understanding**
#
# Here we are going to answer the following three business questions specific to developers in USA
#
# - Which Programming Languages are the most Popular in the USA?
# - Which programming language is the most popular among young developers?
# - What programming language do developers who earn top 5 average Salaries use?
#
# Answering the above business questions helps those new the field to make a better choice on which programming language to learn based on popularity and how much they can earn if they learn a particular programming language.

# ### Data Understanding
# **Step 1: A look at  all the data**

# Data we are using for this analysis is Stack Overflow Annual Developer Survey Data
#
# More than 100K responses fielded from 183 countries and dependent territoris. Many thanks to [Stack Overflow](https://insights.stackoverflow.com/survey) for making this data publicly available. However, I have selected developers in USA to answer the business questions stipulated above.
#
# before diving into answering the above questions, let's have a look at the data.

#import packages
import pandas as pd
from collections import defaultdict
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
# %matplotlib inline

# read all the data
survey_df = pd.read_csv('data/survey_results_public.csv', low_memory=False)
survey_df.head()

# read the schema associated to the data
survey_schema = pd.read_csv('data/survey_results_schema.csv')
survey_schema.head()

# developers per country
dev_per_country  = survey_df[['Country', 'Respondent']].groupby('Country').count().sort_values(by='Respondent', ascending=False)
dev_per_country.iloc[:10,:].plot(kind='bar', figsize=(10,5))
plt.title('Developers by Country')
plt.xlabel('Country')
plt.ylabel('Developer Count');

# The above data shows the highest number of developers who responded to the survey reside in USA. Our business questions also focus on those developers only. I have left the analysis comparing the result with other countries to you the reader.

# **Step 2: A look at the US developers data**

# How much percent of the total developers currently reside in the USA?
us_survey = survey_df[survey_df.Country=='United States']
print('{0:.2f}% developers  are in the USA'.format(100*us_survey.shape[0]/survey_df.shape[0]))

# As shown int the cell below we are going to use the column **LanguageWorkedWith** and **Salary** to aswner our business questions.

list(survey_schema[survey_schema['Column']=='LanguageWorkedWith'].QuestionText)

list(survey_schema[survey_schema['Column']=='Salary'].QuestionText)

print("{0:.2f}% responders didn't specify the Language they worked with".
      format(100*us_survey.LanguageWorkedWith.isnull().sum()/us_survey.shape[0]))

print("{0:.2f}% responders haven't disclosed Salary".
      format(100*us_survey.Salary.isnull().sum()/us_survey.shape[0]))

# We can see above that LanguageWorkedWith and Salary fields are null but those won't have any impact in this analysis as I have only considered those who have specified thos entries as shown in below analysis.

# ### Answering Business quesions
# **1. What are the top most programming languages developers worked with?**
#
# To answer this question we need LanguageWorkedWith column, find all the possible programming languages included in the survey and count each of them to find the total.
#
# **1.1-Data Preparation**

lang_df = us_survey.LanguageWorkedWith.value_counts().reset_index()
lang_df.rename(columns={'index':'Language', 'LanguageWorkedWith':'count'}, inplace=True)
lang_df.head()

all_values = us_survey.LanguageWorkedWith.values
expanded_values = []
for x in all_values:  
    expanded_values.extend(str(x).split(';'))
lang_list = set(expanded_values)
print(len(lang_list))
print(lang_list)


#Now we want to see how often each of these individual values appears - I wrote 
# this function to assist with process - it isn't the best solution, but it gets
# the job done and our dataset isn't large enough to computationally hurt us too much.
# Thanks to https://github.com/jjrunner/ for the following function
def total_count(df, col1, col2, look_for):
    '''
    INPUT:
    df - the pandas dataframe you want to search
    col1 - the column name you want to look through
    col2 - the column you want to count values from
    look_for - a list of strings you want to search for in each row of df[col]
    
    OUTPUT:
    new_df - a dataframe of each look_for with the count of how often it shows up 
    '''
    new_df = defaultdict(int)
    for val in look_for:
        for idx in range(df.shape[0]):
            if val in df[col1][idx]:
                new_df[val] += int(df[col2][idx])   
    new_df = pd.DataFrame(pd.Series(new_df)).reset_index()
    new_df.columns = [col1, col2]
    new_df.sort_values('count', ascending=False, inplace=True)
    return new_df


lang_df_sep_count = total_count(lang_df, 'Language', 'count', lang_list)
lang_df_sep_count.head()

lang_df_sep_count['perc'] = 100*lang_df_sep_count['count']/np.sum(lang_df_sep_count['count'])
lang_df_sep_count.head()

# **1.2. Answer to our first business question**

fig, ax = plt.subplots(1,1)
fig.set_figwidth(20)
x = np.arange(1,39)
# Set number of ticks for x-axis
ax.set_xticks(x)
ax.bar(x, lang_df_sep_count.perc)
x_tick_labels = lang_df_sep_count.Language
plt.title('Percentage of Programming Language Users for All Ages');
plt.xlabel('Programming Language');
plt.ylabel('Count');
ax.set_xticklabels(x_tick_labels, rotation='vertical', fontsize=10);

# The above graph shows the top programming languages developers worked with are C, Java, JavaScript, HTML and CSS. Actual numbers show C and Java are pretty close. Not surprisingly, users of HTML and JavaScipt are very close since there is a high chance to use JavaScript for developering working with HTML and CSS. Now let's look at how usage varies by age.

# **2. Which programming language is the most popular among young developers**
#
# Here we'll look at programming language that are poppular among 18-24 years old age range.
#
# **2.1 Data preparation**
#
# Here we are going to create a new entry for each programming language

#df_lang = survey_df[['Respondent', 'Age', 'LanguageWorkedWith']]
df_lang_exp = pd.concat([survey_df.Respondent, survey_df.Age, survey_df.LanguageWorkedWith.str.split(';')], axis=1).dropna()

df_stacked = pd.DataFrame({'Respondent':np.repeat(df_lang_exp.Respondent.values, 
                                                    df_lang_exp.LanguageWorkedWith.str.len()),
                             'Age':np.repeat(df_lang_exp.Age.values, 
                                                    df_lang_exp.LanguageWorkedWith.str.len()),
                        'LanguageWorkedWith':np.concatenate(df_lang_exp.LanguageWorkedWith.values)
                            })
df_stacked.head()

# +
from wordcloud import WordCloud

lang = df_stacked["LanguageWorkedWith"].value_counts().reset_index()
wrds = lang["index"].str.replace(" ","")
wc = WordCloud(background_color='white', colormap=plt.cm.viridis, scale=5).generate(" ".join(wrds))
plt.figure(figsize=(10,10))
plt.imshow(wc, interpolation="bilinear")
plt.axis("off")
plt.title("Word Cloud of Programming Languages in USA\n",
          fontdict={'size':16, 'weight': 'bold'});
# -

df_stacked['Age_lang_Count'] =1  
groupby_age_language  = df_stacked.groupby(['Age','LanguageWorkedWith'])
groupby_age_language = groupby_age_language['Age_lang_Count'].aggregate(np.sum).unstack()

groupby_scaled = groupby_age_language.div(groupby_age_language.sum(axis=1), axis=0)
groupby_scaled.head()

# **2.1 Answers to our second business question**

groupby_scaled.iloc[0, :].plot(kind = 'bar', 
                          title = 'Programming Language Use for 18 - 24 years olds', 
                          figsize=(15,5))
plt.ylabel('Counts')
plt.show();

plt.figure(figsize=(20,20))
groupby_scaled.iloc[-1, :].plot(kind = 'bar', 
                          title = 'Programming Language Use for Under 18 years olds', 
                          figsize=(15,5))
plt.ylabel('Counts')
plt.show();

# Leading programming languages in this age group : HTML, JavaScript, CSS, SQL; followed by Java, Python and PHP.As we can see HTML, JavaScript and CSS go hand in hand as anyone working as web developer should have skills related to those languages.

groupby_scaled.iloc[:, [4, 17,27]].plot(kind = 'bar', 
                          title = 'Programming Language -  C++, Python and Java by Age', 
                          figsize=(15,10))
plt.ylabel('Counts')

# Looking at the three popular programming languages in the above plot, we can see that Python is leading in the under 18 years old age range. We can see a trend in Python users growing steadily.

# **3.What programming language do developers who earn top 5 average Salaries use?**
#
# **3.1 Data preparation**

df_lang_sal = pd.concat([survey_df.Respondent, survey_df.Salary, survey_df.LanguageWorkedWith.str.split(';')], axis=1).dropna()

df_stacked_sal = pd.DataFrame({'Respondent':np.repeat(df_lang_sal.Respondent.values, 
                                                    df_lang_sal.LanguageWorkedWith.str.len()),
                             'Salary':np.repeat(df_lang_sal.Salary.values, 
                                                    df_lang_sal.LanguageWorkedWith.str.len()),
                        'LanguageWorkedWith':np.concatenate(df_lang_sal.LanguageWorkedWith.values)
                            })
df_stacked_sal.head()

df_stacked_sal['Salary'] = df_stacked_sal['Salary'].replace({'\,':''}, regex=True).astype(float)

#removing those with unsually high salary to make this analysis more realistic
MAX_LIM = df_stacked_sal.loc[:,'Salary'].quantile(0.95)
df_stacked_sal['Salary'] = df_stacked_sal.loc[df_stacked_sal['Salary'] < MAX_LIM, 'Salary']
#remove those with zero Salary
df_stacked_sal = df_stacked_sal[df_stacked_sal.Salary > 0]

groupby_lang_sal  = df_stacked_sal.groupby(['LanguageWorkedWith'])['Salary'].aggregate(np.mean)
groupby_lang_sal.head()

# **3.2 Answer to our third business question**

groupby_lang_sal.plot(kind = 'bar', 
                          title = 'Programming Language vs Average Salary', 
                          figsize=(15,5))
plt.ylabel('Average Salary')
plt.show();

# ### Results Evaluation
# **1. Which Programming Languages are the most Popular in the USA?**
#
# The above graph shows the top programming languages developers worked with are C, Java, JavaScript, HTML and CSS. Actual numbers show C and Java are pretty close. Not surprisingly, users of HTML and JavaScipt are very close since there is a high chance to use JavaScript for developering working with HTML and CSS.
#
# **2. Which programming language is the most popular among young developers?**
#
# Leading programming languages in this age group : HTML, JavaScript, CSS, SQL; followed by Java, Python and PHP.As we can see HTML, JavaScript and CSS go hand in hand as anyone working as web developer should have skills related to those languages.
#
# **3. What programming language do developers who earn top 5 average Salaries use?**
#
# We have found out that Clojure, Go, Hack, Objective-C, Ocaml, Scala are the top of the list interms of average Salary.
#

StarcoderdataJupyterScriptsDedupFiltered