import pandas as pd
import numpy as np

data = pd.DataFrame({'k1': ['one'] * 3 + ['two'] * 4,
                  'k2': [1, 1, 2, 3, 3, 4, 4]})
data

# The DataFrame method duplicated returns a boolean Series indicating whether each
# row is a duplicate or not

data.duplicated()

#  drop_duplicates  drop the duplicates
data.drop_duplicates()

# Both of these methods by default consider all of the columns; 
# until we defined a specific column name
data.drop_duplicates(['k1'])

data.drop_duplicates(['k2'])

# duplicated and drop_duplicates by default keep the first observed value combination.
# Passing take_last=True will return the last one:

data.drop_duplicates(['k1', 'k2'], take_last=True)

data.drop_duplicates(['k1', 'k2'], keep='last')

"""
Explanation: Data Transformation
1. Removing Duplicates
End of explanation
"""


data = pd.DataFrame({'food': ['bacon', 'pulled pork', 'bacon', 'Pastrami',
                               'corned beef', 'Bacon', 'pastrami', 'honey ham',
                               'nova lox'],
                      'ounces': [4, 3, 12, 6, 7.5, 8, 3, 5, 6]})
data

meat_to_animal = {
  'bacon': 'pig',
  'pulled pork': 'pig',
  'pastrami': 'cow',
  'corned beef': 'cow',
  'honey ham': 'pig',
  'nova lox': 'salmon'
}
meat_to_animal

data['animal'] = data['food'].map(str.lower).map(meat_to_animal)
data

data['food'].map(lambda x: meat_to_animal[x.lower()])

"""
Explanation: 2. Transforming Data Using a Function or Mapping
End of explanation
"""


data = pd.Series([1., -999., 2., -999., -1000., 3.])
data

# The -999 values might be sentinel values for missing data. To replace these with NA
# values that pandas understands, we can use replace

data.replace(-999, np.nan)

# replace multiple values at once

data.replace([-999,-1000],  np.nan)

# To use a different replacement for each value, pass a list of substitutes:
data.replace([-999,-1000],  [np.nan, 0])

# The argument passed can also be a dict:
data.replace({-999: np.nan, -1000: 0})

"""
Explanation: Replacing Values
End of explanation
"""


data = pd.DataFrame(np.arange(12).reshape((3, 4)),
                     index=['Ohio', 'Colorado', 'New York'],
                     columns=['one', 'two', 'three', 'four'])
data

data.index.map(str.upper)

data.index = data.index.map(str.upper)
data

# o create a transformed version of a data set without modifying the original,
# a useful method is rename:
data.rename(index=str.title, columns=str.upper)

#  rename can be used in conjunction with a dict-like object providing new values
#  for a subset of the axis labels

data.rename(index={'OHIO': 'INDIANA'},
                columns={'three': 'peekaboo'})

# rename saves having to copy the DataFrame manually and assign to its index and columns 
# attributes. Should you wish to modify a data set in place, pass inplace=True

_ = data.rename(index={'OHIO': 'INDIANA'}, inplace=True)
data

"""
Explanation: Renaming Axis Indexes
End of explanation
"""


# Continuous data is often discretized or otherwised separated into “bins” for analysis.

ages = [20, 22, 25, 27, 21, 23, 37, 31, 61, 45, 41, 32]
ages

bins = [18, 25, 35, 60, 100]
bins

cats = pd.cut(ages, bins)
cats

cats.labels

cats.codes

[bins[i] for i in cats.codes]

cats.levels

cats.categories

pd.value_counts(cats)

# Consistent with mathematical notation for intervals, a parenthesis means that the side
# is open while the square bracket means it is closed (inclusive). Which side is closed can
# be changed by passing right=False:
pd.cut(ages, [18, 26, 36, 61, 100], right=False)

# You can also pass your own bin names by passing a list or array to the labels option:
group_names = ['Youth', 'YoungAdult', 'MiddleAged', 'Senior']
pd.cut(ages, bins, labels=group_names)

# If you pass cut a integer number of bins instead of explicit bin edges, it will compute
# equal-length bins based on the minimum and maximum values in the data. 
data = np.random.rand(20)
data

cat=pd.cut(data, 4, precision=2)
cat

cat.categories

cat.codes

pd.value_counts(cat)

cat=pd.cut(data, 7, precision=2)
cat

pd.value_counts(cat)

# A closely related function, qcut, bins the data based on sample quantiles.
data = np.random.randn(1000)

cats = pd.qcut(data, 4) # Cut into quartiles
cats

pd.value_counts(cats)

# Similar to cut you can pass your own quantiles (numbers between 0 and 1, inclusive):
cat=pd.qcut(data, [0, 0.1, 0.5, 0.9, 1.])
cat

pd.value_counts(cat)

"""
Explanation: Discretization and Binning
End of explanation
"""


# Filtering or transforming outliers is largely a matter of applying array operations.

np.random.seed(12345)

data = pd.DataFrame(np.random.randn(1000, 4))
data.describe()

# Suppose you wanted to find values in one of the columns exceeding three in magnitude:
col = data[3]

col[np.abs(col) > 3]

# To select all rows having a value exceeding 3 or -3, you can use the any method on a
# boolean DataFrame:

 data[(np.abs(data) > 3).any(1)]

# code to cap values outside the interval -3 to 3
data[np.abs(data) > 3] = np.sign(data) * 3

data.describe()

# data[np.abs(data) > 3]

"""
Explanation: Detecting and Filtering Outliers
End of explanation
"""


# Permuting (randomly reordering) a Series or the rows in a DataFrame is easy to do using
# the numpy.random.permutation function. Calling permutation with the length of the axis
# you want to permute produces an array of integers indicating the new ordering:

df = pd.DataFrame(np.arange(5 * 4).reshape(5, 4))
df

sampler = np.random.permutation(5)
sampler

df.take(sampler)

# To select a random subset without replacement, one way is to slice off the first k ele-
# ments of the array returned by permutation, where k is the desired subset size. There
# are much more efficient sampling-without-replacement algorithms, but this is an easy
# strategy that uses readily available tools

df.take(np.random.permutation(len(df))[:3])

# To generate a sample with replacement, the fastest way is to use np.random.randint to
# draw random integers:
bag = np.array([5, 7, -1, 6, 4])

sampler = np.random.randint(0, len(bag), size=10)
sampler

draws = bag.take(sampler)
draws

"""
Explanation: Permutation and Random Sampling
End of explanation
"""


df = pd.DataFrame({'key': ['b', 'b', 'a', 'c', 'a', 'b'],
                    'data1': range(6)})
df

pd.get_dummies(df['key'])

dummies = pd.get_dummies(df['key'], prefix='key')
dummies

df_with_dummy = df[['data1']].join(dummies)
df_with_dummy

# reading the movie data
movies = pd.read_csv("dataset/movies.csv")
movies.head()

# Adding indicator variables for each genre requires a little bit of wrangling. First, we
# extract the list of unique genres in the dataset (using a nice set.union trick)

genre_iter = (set(x.split('|')) for x in movies.genres)
genres = sorted(set.union(*genre_iter))

# for i in genre_iter:
#    print(i)

genres

dummies = pd.DataFrame(np.zeros((len(movies), len(genres))), columns=genres)

for i, gen in enumerate(movies.genres):
        dummies.ix[i, gen.split('|')] = 1

movies_windic = movies.join(dummies.add_prefix('Genre_'))

movies_windic.ix[0]

"""
Explanation: Computing Indicator/Dummy Variables
Another type of transformation for statistical modeling or machine learning applica-
tions is converting a categorical variable into a “dummy” or “indicator” matrix. If a
column in a DataFrame has k distinct values, you would derive a matrix or DataFrame
containing k columns containing all 1’s and 0’s. pandas has a get_dummies function for
doing this, though devising one yourself is not difficult. Let’s return to an earlier ex-
ample DataFrame
End of explanation
"""


