numbers = range(1000)

def doubled_sum(values):
    total = 0
    for n in numbers:
        total += n*2
    return total

print doubled_sum(numbers)

"""
Explanation: Map Reduce Explained
Is a programming model invented by Google to perform distributed computation on huge amounts of data, it's heavily inspired by the functional programming map and reduce functions.
The process is divided in two steps:
 - A mapping step: the input is divided in subparts until it reaches its minimum processable particle which gets processed and the corresponding result gets returned.
 - A reduction step: each value processed by the mapping step goes back to the reduction step that joins the values together to a single result.
<img src="files/MapReduce.jpeg" width="640"/>
Practice Example
<img src="files/MapReduce_example.png"/>
Example Problem without MapReduce
Our example problem is to compute the sum of the doubles of various numbers.
So if we have 1, 2, 3 as the input the result would be 2+4+6 -> 12
End of explanation
"""


numbers = range(1000)

def mapper(value):
    return value*2

def reducer(*values):
    return sum(values)
#first_step = map(mapper, numbers)
#print first_step
result = reduce(reducer, map(mapper, numbers))
print result

"""
Explanation: Example Problem using MapReduce
As Python implements the functional programming paradigm too, it provides the functions required to implement the map-reduce paradigm builtin.
The foundation tools to implement map reduce are:

"mapper" which is in charge of mapping each input value to a corresponding output value
"reducer" which is in charge of merging multiple mapper outputs into a single output. 

Both phases can be called multiple times (the output of a reducer can become the input of another reducer and a mapper can call other mappers).
Many MapReduce implementations also have additional phases like "combination" and "aggregation" which are executed after the mapper or the reducer to further cleanup their output.
End of explanation
"""


from itertools import islice
import multiprocessing


class ParallelMapReduce(object):
    def __init__(self, map_func, reduce_func, num_workers=None):
        self.num_workers = num_workers
        self.map_func = map_func
        self.reduce_func = reduce_func
        self.pool = multiprocessing.Pool(num_workers)
    
    def partition(self, n, iterable):
        i = iter(iterable)
        piece = list(islice(i, n))
        while piece:
            yield piece
            piece = list(islice(i, n))
    
    def __call__(self, inputs):
        values = self.pool.map(self.map_func, inputs)
        
        print '>>> MAPPED VALUES (%s values): %s, ...' % (len(values), str(values[:10]))

        values = self.pool.map(self.reduce_func, 
                               self.partition(len(values)//self.num_workers, values))
        print '>>> REDUCED VALUES', values

        return self.reduce_func(values)

"""
Explanation: The previous map-reduce in pure python implementation lacks of course the core feature of MapReduce: working parallely.
It's easy to understand that as each mapper and reducer works only on a subset of the data (its own input) it can work independently from the status of the other mappers and reducers. So the computation can proceed parallely.
Parallel Map Reduce in Pure Python
It's really easy to simulate a parallel map reduce in python by using the multiprocessing module
End of explanation
"""


numbers = range(1000)

def mapper(value):
    return value*2

def reducer(values):
    return sum(values)

mapreduce = ParallelMapReduce(mapper, reducer, 10)
print mapreduce(numbers)

"""
Explanation: The previous mapreduce implementation takes a Mapper and a Reducer and splits them across num_workers until it gets back the final result
End of explanation
"""


