# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
# module 5
# in this module, we will learn how to update rows and columns and also modifying data within our dataframe.

# lets look at updating column first, then we will update some rows.

# +
import pandas as pd

mul_people={
    "first":["Ahammad","Corey","John","Jaden","Boss"],
    "last":["Shawki","Schafer","Joker","Joker","Zack"],
    "email":["<EMAIL>","<EMAIL>",
             "<EMAIL>","<EMAIL>","<EMAIL>"],
}

dic_df=pd.DataFrame(mul_people)
dic_df
# -

# before we modify the columns, lets first look at these using columns attribute.
print(dic_df.columns)

# +
# lets say we want to update the column name to be more specific.
# we want "firstname" instead of first column and things like that. 

# there are couple of ways that we can do that.
# so first if we want to rename all of the columns, then we can simply do an assignment using the columns attribute.
# so, in order to do that-
dic_df.columns=['first name', 'last name', 'email', 'salary']
# -

dic_df

# +
# this is useful when we are passing different names for all of our columns.
# but we may need to change the name of few different columns.
# but one thing that is common is the need to change something specific about each column in our dataframe.
# for example,if our column names are all uppercase and we want to make them lowercase or vice verse.
# or we want to replace our column name spaces with underscore.
# in this case we can use a list comprehension.

# lets say we want to uppercase all of the column namas here.
new_columns=[name.upper() for name in dic_df.columns]
dic_df.columns=new_columns
# -

dic_df

# Remove spaces with undersoce.
# sometimes we need to remove all the spaces in our column name.
# it is useful because we cant call them with dot(.) method if there are spaces.
#dic_df.FIRST NAME # this will give us an error.
# so, we can-
new_columns=[name.replace(" ","_") for name in dic_df.columns]
dic_df.columns=new_columns

dic_df

# or we can use this str method in our dic_df.columns.
dic_df.columns=dic_df.columns.str.replace(" ","_")
dic_df
# this will do the same thing

# let us go ahead and set those columns the way they were before.
new_columns=[name.lower() for name in dic_df.columns]
dic_df.columns=new_columns

dic_df

# +
# so what we have seen is that how to modify all of our columns.
# but what if we want to change some specific columns.
# we can use the rename() method and just pass a dictionary of the column that we want to change.

# if we want to change the first_name and last_name what they were before we can-
dic_df.rename(columns={"first_name":"first","last_name":"last"},inplace=True)
# here the key will be the old name, and the value will be the new name.
# if we run this now, it might look it work.
# but if we look at our dataframe, those changes wont be work.
# so if we want that change to be placed permanently then we need to say inplace=True.
# -

dic_df

# +
# now lets look at updating our data in the rows.
# and we will spent the majority of time learning how to update data in our row
# since there are so much that we can do here.

# first lets look at how to update a single value.
# we can access values in rows using loc[] and iloc[] indexers.
# let us grab the third row and change its last name to smith.
dic_df.loc[2]
# -

# now in order to update the information, there are couple of ways that we can do this.
# we can pass in all of the new values of this row by passing in a list.
dic_df.loc[2] = ["John","Smith","<EMAIL>",33600]

dic_df

# but what if we have a lot of columns and we wanted to change a couple of values?
# in this case we can just specify the exact column using loc[].
dic_df.loc[4,["first","email"]]=["Zakaria","<EMAIL>"]

dic_df

# if we want to change a single value we dont need to pass a list.
dic_df.loc[3,"first"]="Jackson"

dic_df

# now pandas does have another indexer called at.
# this is specifically made for changing or grabbing specific value.
# but .loc[] indexer is recommanded.
dic_df.at[3,"first"]="Real"

dic_df

# +
# now we will look at a mistake which is very common.
# that is many people try to change the value without using a indexer.
# lets say we have a large dataframe where we want to find a man with a email of <EMAIL> 
# and we want to change his last name.

# first grab the mans info using filter.
filt=(dic_df["email"]=="<EMAIL>")
dic_df[filt]
# -

# that actually return a dataframe there that has a single row.
# we can grab the last name column just by accessing the "last" column after the dataframe in brackets[]
dic_df[filt]["last"]

# now we can say that, ok we have got the value. lets change that using equal sign.
dic_df[filt]["last"]="Hector"
# we are getting a big warning: SettingWithCopyWarning
# they are suggested to-
# Try using .loc[row_indexer,col_indexer] = value instead

dic_df
# if we see our dataframe, we can see that it is not changed.
# the reason it didn't do any change here is little complicated.
# essentially the way we are doing it here requirs multiple operation on the backgrounds,
# which cant determain whether pandas return a view or a copy of our dataframe.
# so when our value isnt getting set, its because it is getting set on a temporary object 
# which is getting tossed out immidiately after
# pandas is giving a little link here of the documentation directly within its warning itself.
# if we want to know more about this then we can click on that link and learn it in details.
# but the moral of the story here is that when we are setting or changing values,
# we should use .loc[], .iloc[] or .at[] indexer that we have already seen.
# again loc[] and iloc[] indexers are recommanded.
dic_df.loc[filt,"last"]="Hector"

dic_df
# we can see that it works fine.
# so whenever we see that kind of stuffs we mustn't ignore them.
# instead we should learn more about them and be careful in future,

# +
# we have learned how to update a single row of data.
# but how to update multiple rows of data?
# there are couple of different ways that we can do that.

# lets say we want to change all of the email adderesses so that they are lowercase.
# we can do that with str.lower() method.
# first we need to grab the email column.
dic_df["email"].str.lower()
# this  just returns lowercase values of those emails.
# it didnt actually make that change.
# in order to make that change we can simply assign the "email" column to this values.
dic_df["email"]=dic_df["email"].str.lower()
# -

dic_df

# +
# that's one way that we can change multiple rows at once.
# but maybe we want to do something a little more advanced.
# there are several ways to do this
# and we will go over all four popular methods in order to do this.
# lot of people get this four methods confused.
# so lets go over each one individually and try to explain them in details.
# and there is definately a good reason why all of these people get confused
# because there are very simmilar in what they do.

# +
# the 4 methods that we will talk about are-
# apply
# map
# applymap
# replace

# +
# apply:
# apply is use for calling a function on our values.
# apply can work in either a dataframe or a series object.
# its behaviour might be different than we expect for each of different objects.

# first, lets look how apply works in a series? 
# when we use this on a series it can apply function on every values on our series.
# lets say we want to see the length of all of our email addresses.
# we can use the apply function.
dic_df["email"].apply(len)
# here we need to use the name of len funtion without parenthesis as a arguement of apply function.
# because we are not executing the function. So, no brackets()

# +
# so apply is useful to apply functions on all our grabbed values.
# we can also use the apply function for updating values as well.

# let us write a simple function here.
def update_email(email):
    return email.upper()

# lets apply this function to our email column
dic_df["email"]= dic_df["email"].apply(update_email)
# -

dic_df

# for simple function like this people also use lambda function as an arguement in apply function.
dic_df["email"]= dic_df["email"].apply(lambda x:x.lower())

dic_df

# +
# so if we are more comfortable writting with the general function, then we can do it without lambda function.
# and we can use lambda function for simple function like this.

# now here we are working with strings.
# but we can use the apply function with any other datatypes 
# like integers or float to calculate all the values at once.

# this is how apply works in series object.
# now lets see how apply works in dataframe.
# when we run apply on a series, it will apply in all values of the series.
# when we run apply on a dataframe, it run a function on each row and column of that dataframe.
# lets see what this looks like.

# instead of writing a specific column,if we use apply() method in the dataframe.
# it will run the function on every column of our dataframe.
dic_df.apply(len)
# in this case, it is giving us our "first" column has 5 values and so on.
# thats just the number of rows in each column.
# but that we didnt expect.
# -

# so whats going on here, is that is not applying the function to all values of our dataframe.
# it is actually applying the function to each series(column) of that dataframe.
# we can also use the apply() function to our rows if we change the axis arguement to "columns".
dic_df.apply(len,axis="columns")

# +
# so basically, we want to use functions that will make sense to be used on series object
# when using apply on a dataframe.

# for example lets say we want to grab the minimum value from each column.
# Series object (which is from pandas library) has a min() method.
# so we could pass that method in apply and see the minimum values of each series.
# when we try to grab the minimum value from a series of strings,
# it will just give us the value which is first in alphabetical orger.
dic_df.apply(pd.Series.min)
# -

# and we can use lambda function for this as well.
# here the param of lambda is going to be a series if we apply that lambda on a dataframe.
dic_df.apply(lambda x: x.min())
# a series have a min() method.
# if we run that we can see that will give us the same thing that we run before.

# +
# this is way more useful if our dataframe contains numeric data.
# for example we can use numpy to apply this function
# to all of our series object or any time of numerical analysis like that.

# ok if we run apply in series, it will apply a function to all of the values of that series.
# again if we run apply in database, it will apply a function to all of the columns of that database.

# but we might be wondering if there is a way
# to apply a function on all of the each individual values of a database?
# and thats what applymap() function is use for.

# +
# applymap():
# applymap only works in the dataframe.
# series object dont have the applymap() method.

# lets use that built_in len() function and apply that using applymap()
dic_df.applymap(len)

# +
# we can see that its now applying the function to each individual value of our dataframe.
# now we know the difference between apply() and applymanp()
# we should be careful when we need to use one of them.

# for example, since we have a dataframe full of strings here.
# if we wanted all the values will be lowercase.
# we can use applymap and string lower method.
dic_df.applymap(str.lower)

# +
# if we have numeric data in our dataframe 
# we will get an error here since we cant use str methods in numeric values.
# we need to pass in a more complicated function so that they can handle it appropriately.

# now lets look at the map method.
# map():
# map method only works in a series.
# map is used for substituting each value in a series with another value.
# lets say we want to substitute couple of values from our "first" column.
dic_df["first"].map({"Corey":"Mooray","Boss":"Coaster"})
# here we need to add a dictionary as an arguement.

# +
# if we run this we can see that it prints a series where the given names are substituted out.
# the values that we didnt sunstitute is converted is turned in NaN value(not an number)
# that may or may not be we want.

# in this example, we likely want to get rid  of this NaN values.
# we might think to keep john and substitute other names.
# for this we can use replace() method.
dic_df["first"].replace({"Corey":"Mooray","Boss":"Coaster"})

# +
# everything that we done here, doesn't actually change the dataframe.
# if we want to set this to the actual column, then we will have to do something like this.
dic_df["first"]=dic_df["first"].replace({"Corey":"Mooray","Boss":"Coaster"})

# actually, the difference between map() and replace method is 
# replace() only replace the values that we want and keeps other values unchanged.
# on the other hand, map() also replace the values that we want but it converts other values,
# which we havent indicated, into nan values. So we need to be careful using these functions.
# map() in pandas is different that the map() function we have in python.
# python map() mostly works like the apply() function.
# -

dic_df

# +
# now we have look at lot of ways to modify our data.
# now go to our larger dataset and look at some real world example of how apply what we have learned here.

df=pd.read_csv("stackoverflow/survey_results_public.csv",index_col="Respondent")
schema_df=pd.read_csv("stackoverflow/survey_results_schema.csv",index_col="Column")

pd.set_option("display.max_columns", 85)
pd.set_option("display.max_rows", 85)

df.head()
# -

# lets say we want to rename the salary column (which is ConvertedComp) to "SalaryUSD".
# we can do that with the rename() method.
df.rename(columns={"ConvertedComp":"SalaryUSD"},inplace=True)
# after we are assured that everything is okay here, then we can use inplace arguemnet.

# inplace is a good idea.
# when we are working with pandas 
# we have to do lot of renames, filter,update, modify,manipulate and things like that.
# and it is a common scenario that we may mistake.
# so it is better to check if it is made the change as we want.
# and then we can make that change permanent using inplace.
# now lets look if we can access the SalaryUSD column.
df["SalaryUSD"]
# we can see that the change did work.

# lets access the Hobbyist column.
df["Hobbyist"]

# lets say we want to convert the values and convert all of the Yes values to True boolean and
#  convert all of the No values to False boolean using map.
df["Hobbyist"].map({"Yes":True,"No":False})

# now there isnt an inplace arguement for map().
# so what we can do here is that we can just set the series equal to that new series that we have made with map()
df["Hobbyist"]=df["Hobbyist"].map({"Yes":True,"No":False})

df

# +
# remember again when we used map(),
# anything that isn't the dictionay will converted into NaN value.
# if we only want to replace the yes and no values and remaining all of the values untouched,
# then we can use replace() method, instead of map() method.

StarcoderdataJupyterScriptsDedupFiltered