def isEnglish(string):
    try:
        string.encode('ascii')
    except UnicodeEncodeError:
        return False
    else:
        return True

"""
Explanation: P3 - Data Wrangling with MongoDB
OpenStreetMap Project Data Wrangling with MongoDB
Gangadhara Naga Sai<a name="top"></a>
Data used -<a href=https://mapzen.com/metro-extracts/> MapZen Weekly OpenStreetMaps Metro Extracts</a>
Map Areas:
   These two maps are selected since ,right now i am living at Hoodi,Bengaluru. And my dream is to do my masters in japan in robotics,so i had selected locality of University of tokyo, Bunkyo.I really wanted to explore differences between the regions.

<a href=https://mapzen.com/data/metro-extracts/your-extracts/fdd7c4ef0518> Bonkyu,Tokyo,Japan. </a> 
<a href=https://mapzen.com/data/metro-extracts/your-extracts/c1f2842408ac> Hoodi,Bengaluru,india </a>

Working Code :
- <a href=https://mapzen.com/data/metro-extracts/your-extracts/fdd7c4ef0518> Bonkyu,Tokyo,Japan. </a> 
<hr>


Problems Encountered in the Map
Filtering Different Language names
Over-­abbreviated Names
Merging both cities
Data Overview
Additional Data Exploration using MongoDB
Conclusion

<hr>

<h2><a name="problems"></a> **1. Problems Encountered**</h2>


Some of names were in different Languages so ,i had to filter out them and select english names for both maps Hoodi and Bunkyo 
Street names with different types of abbreviations. (i.e. 'Clark Ave SE' or 'Eubank Northeast Ste E-18') 
Two cities have to be accessed from one database

Names in Different Language<a name="Language"></a>
Different regions have different languages ,and we find that someof names were in different language which are filltered to get only english names.
Which would check weather the charecters belong to ascii or not
End of explanation
"""


#the city below can be hoodi or bunkyo
for st_type, ways in city_types.iteritems():
    for name in ways:
        better_name = update_name(name, mapping)
        if name != better_name:
            print name, "=>", better_name

#few examples 
Bunkyo:
Meidai Jr. High Sch. => Meidai Junior High School
St. Mary's Cathedral => Saint Mary's Cathedral
Shinryukei brdg. E. => Shinryukei Bridge East
Iidabashi Sta. E. => Iidabashi Station East
...

Hoodi:
St. Thomas School => Saint Thomas School
Opp. Jagrithi Apartment => Opposite Jagrithi Apartment
...

"""
Explanation: <hr>

Over-­abbreviated Names<a name="abbr"></a>
Since the most of data being manually uploaded, there are lot of abbreviations in street names,locality names.
Where they are filtered and replaced with full names.
End of explanation
"""


bangalore.osm -40MB
bangalore.osm.json-51MB
tokyo1.osm- 82MB
tokyo1.osm.json-102.351MB

"""
Explanation: <hr>

Merging Both cities<a name="combine_cities"></a>
These two maps are selected since ,right now i am living at Hoodi,Bengaluru. And one day i want do my masters in japan in robotics,so i had selected locality of University of tokyo, Bunkyo.I really wanted to explore differences between the regions. 
I need to add a tag named "city" so i can differentiate them from the database.
<hr>

2. Data Overview<a name="data_overview"></a>
This section contains basic statistics about the dataset and the MongoDB queries used to gather them.
File Sizes
End of explanation
"""


print "Bunkyo:",mongo_db.cities.find({'city':'bunkyo'}).count()
print "Hoodi:",mongo_db.cities.find({'city':'hoodi'}).count()

"""
Explanation: Number of documents
End of explanation
"""


print "Bunkyo:",mongo_db.cities.find({"type":"node",
                                    'city':'bunkyo'}).count()
print "Hoodi:",mongo_db.cities.find({"type":"node",
                                    'city':'hoodi'}).count()

Bunkyo: 1051170
Hoodi: 548862

"""
Explanation: Bunkyo: 1268292
Hoodi: 667842

Number of node nodes.
End of explanation
"""


print "Bunkyo:",mongo_db.cities.find({'type':'way',
                                  'city':'bunkyo'}).count()
print "Hoodi:",mongo_db.cities.find({'type':'way',
                                  'city':'hoodi'}).count()

Bunkyo: 217122
Hoodi: 118980

"""
Explanation: Number of way nodes.
End of explanation
"""


print "Constributors:", len(mongo_db.cities.distinct("created.user"))

Contributors: 858

"""
Explanation: Total Number of contributor.
End of explanation
"""


def pipeline(city):
    p= [{"$match":{"created.user":{"$exists":1},
                                          "city":city}},
                 {"$group": {"_id": {"City":"$city",
                                     "User":"$created.user"},
                            "contribution": {"$sum": 1}}},                            
                 {"$project": {'_id':0,
                               "City":"$_id.City",
                               "User_Name":"$_id.User",
                               "Total_contribution":"$contribution"}},
                 {"$sort": {"Total_contribution": -1}},
                 {"$limit" : 5 }]
    return p
result1 =mongo_db["cities"].aggregate(pipeline('bunkyo'))
for each in result1:    
    print(each)
print("\n")
result2 =mongo_db["cities"].aggregate(pipeline('hoodi'))
for each in result2:    
    print(each)


"""
Explanation: <hr>

3. Additional Data Exploration using MongoDB<a name="exploration"></a>
I am going to use the pipeline function to retrive data from the database
End of explanation
"""


pipeline=[{"$match":{"Additional Information.amenity":{"$exists":1},
                                          "city":city}},
                 {"$group": {"_id": {"City":"$city",
                                    "Amenity":"$Additional Information.amenity"},
                            "count": {"$sum": 1}}},
                 {"$project": {'_id':0,
                               "City":"$_id.City",
                               "Amenity":"$_id.Amenity",
                               "Count":"$count"}},
                 {"$sort": {"Count": -1}},
                 {"$limit" : 10 }]

"""
Explanation: The top contributors for hoodi are no where near since bunkyo being a more compact region than hoodi ,there are more places to contribute.
<hr>

To get the top Amenities in Hoodi and Bunkyo
I will be showing the pipeline that will go in the above mentioned "Pipleline" function
End of explanation
"""


    p = [{"$match":{"Additional Information.amenity":{"$exists":1},
                                  "Additional Information.amenity":"place_of_worship",
                                  "city":city}},
                       {"$group":{"_id": {"City":"$city",
                                          "Religion":"$Additional Information.religion"},
                                  "count":{"$sum":1}}},
                       {"$project":{"_id":0,
                                    "City":"$_id.City",
                                    "Religion":"$_id.Religion",
                                    "Count":"$count"}},
                       {"$sort":{"Count":-1}},
                       {"$limit":6}]

"""
Explanation: As compared to hoodi ,bunkyo have few atms,And parking can be commonly found in bunkyo locality
<hr>

popular places of worship
End of explanation
"""


p = [{"$match":{"Additional Information.amenity":{"$exists":1},
                                 "Additional Information.amenity":"restaurant",
                                 "city":city}},
                      {"$group":{"_id":{"City":"$city",
                                        "Food":"$Additional Information.cuisine"},
                                 "count":{"$sum":1}}},
                      {"$project":{"_id":0,
                                  "City":"$_id.City",
                                  "Food":"$_id.Food",
                                  "Count":"$count"}},
                      {"$sort":{"Count":-1}}, 
                      {"$limit":6}]

"""
Explanation: As expected japan is popular with buddism,
but india being a secular country it will be having most of the reglious places of worship,where hinduism being majority
<hr>

popular restaurants
End of explanation
"""


 p = [{"$match":{"Additional Information.amenity":{"$exists":1},
                                 "Additional Information.amenity":"fast_food",
                                 "city":city}},
                      {"$group":{"_id":{"City":"$city",
                                        "Food":"$Additional Information.cuisine"},
                                 "count":{"$sum":1}}},
                      {"$project":{"_id":0,
                                  "City":"$_id.City",
                                  "Food":"$_id.Food",
                                  "Count":"$count"}},
                      {"$sort":{"Count":-1}}, 
                      {"$limit":6}]

"""
Explanation: {u'Count': 582, u'City': u'bunkyo'}
{u'Food': u'japanese', u'City': u'bunkyo', u'Count': 192}
{u'Food': u'chinese', u'City': u'bunkyo', u'Count': 126}
{u'Food': u'italian', u'City': u'bunkyo', u'Count': 69}
{u'Food': u'indian', u'City': u'bunkyo', u'Count': 63}
{u'Food': u'sushi', u'City': u'bunkyo', u'Count': 63}
{u'Count': 213, u'City': u'hoodi'}
{u'Food': u'regional', u'City': u'hoodi', u'Count': 75}
{u'Food': u'indian', u'City': u'hoodi', u'Count': 69}
{u'Food': u'chinese', u'City': u'hoodi', u'Count': 36}
{u'Food': u'international', u'City': u'hoodi', u'Count': 24}
{u'Food': u'Andhra', u'City': u'hoodi', u'Count': 21}
Indian style cusine in Bunkyo seems famous, Which will be better if i go to japan and do my higher studies there.
<hr>

popular fast food joints
End of explanation
"""


    p = [{"$match":{"Additional Information.amenity":{"$exists":1},
                                 "Additional Information.amenity":"atm",
                                 "city":city}},
                      {"$group":{"_id":{"City":"$city",
                                        "Name":"$Additional Information.name:en"},
                                 "count":{"$sum":1}}},
                      {"$project":{"_id":0,
                                  "City":"$_id.City",
                                  "Name":"$_id.Name",
                                  "Count":"$count"}},
                      {"$sort":{"Count":-1}}, 
                      {"$limit":4}]

"""
Explanation: Burger seems very popular among japanese in fast foods,i was expecting ramen to be more popular
, but in hoodi pizza is really common,being a metropolitan city.
<hr>

ATM's near locality
End of explanation
"""


##  Martial arts or Dojo Center near locality
import re

pat = re.compile(r'dojo', re.I)
d=mongo_db.cities.aggregate([{"$match":{ "$or": [ { "Additional Information.name": {'$regex': pat}}
                                                   ,{"Additional Information.amenity": {'$regex': pat}}]}}
                            ,{"$group":{"_id":{"City":"$city"
                             , "Sport":"$Additional Information.name"}}}])
for each in d:    
    print(each)

bunkyo: 
{u'_id': {u'City': u'bunkyo', u'Sport': u'Aikikai Hombu Dojo'}}
{u'_id': {u'City': u'bunkyo', u'Sport': u'Kodokan Dojo'}}

hoodi:
{u'_id': {u'City': u'hoodi', u'Sport': u"M S Gurukkal's Kalari Academy"}}


"""
Explanation: There are quite a few ATM in Bunkyo as compared to hoodi
<hr>

Martial arts or Dojo Center near locality
End of explanation
"""


 p = [{"$match":{"Additional Information.shop":{"$exists":1},
                                          "city":city}},
                       {"$group":{"_id":{"City":"$city",
                                  "Shop":"$Additional Information.shop"},
                           "count":{"$sum":1}}},
                       {"$project": {'_id':0,
                                     "City":"$_id.City",
                                     "Shop":"$_id.Shop",
                                     "Count":"$count"}},
                       {"$sort":{"Count":-1}},
                       {"$limit":10}]

{u'Shop': u'convenience', u'City': u'bunkyo', u'Count': 1035}
{u'Shop': u'clothes', u'City': u'bunkyo', u'Count': 282}
{u'Shop': u'books', u'City': u'bunkyo', u'Count': 225}
{u'Shop': u'mobile_phone', u'City': u'bunkyo', u'Count': 186}
{u'Shop': u'confectionery', u'City': u'bunkyo', u'Count': 156}
{u'Shop': u'supermarket', u'City': u'bunkyo', u'Count': 150}
{u'Shop': u'computer', u'City': u'bunkyo', u'Count': 126}
{u'Shop': u'hairdresser', u'City': u'bunkyo', u'Count': 90}
{u'Shop': u'electronics', u'City': u'bunkyo', u'Count': 90}
{u'Shop': u'anime', u'City': u'bunkyo', u'Count': 90}


{u'Shop': u'clothes', u'City': u'hoodi', u'Count': 342}
{u'Shop': u'supermarket', u'City': u'hoodi', u'Count': 129}
{u'Shop': u'bakery', u'City': u'hoodi', u'Count': 120}
{u'Shop': u'shoes', u'City': u'hoodi', u'Count': 72}
{u'Shop': u'furniture', u'City': u'hoodi', u'Count': 72}
{u'Shop': u'sports', u'City': u'hoodi', u'Count': 66}
{u'Shop': u'electronics', u'City': u'hoodi', u'Count': 60}
{u'Shop': u'beauty', u'City': u'hoodi', u'Count': 54}
{u'Shop': u'car', u'City': u'hoodi', u'Count': 36}
{u'Shop': u'convenience', u'City': u'hoodi', u'Count': 36}

The general stores are quite common in both the places

"""
Explanation: I wanted to learn martial arts , 
In japan is known for its akido and other ninjistsu martial arts , where i can find some in bunkyo
Where as in hoodi,india Kalaripayattu Martial Arts are one of the ancient arts that ever existed.
<hr>

most popular shops.
End of explanation
"""


    p = [{"$match":{"Additional Information.shop":{"$exists":1},
                           "city":city,
                           "Additional Information.shop":"supermarket"}},
                       {"$group":{"_id":{"City":"$city",
                                  "Supermarket":"$Additional Information.name"},
                           "count":{"$sum":1}}},
                       {"$project": {'_id':0,
                                     "City":"$_id.City",
                                     "Supermarket":"$_id.Supermarket",
                                     "Count":"$count"}},
                       {"$sort":{"Count":-1}},
                       {"$limit":5}]

{u'Count': 120, u'City': u'bunkyo'}
{u'Count': 9, u'City': u'bunkyo', u'Supermarket': u'Maruetsu'}
{u'Count': 3, u'City': u'bunkyo', u'Supermarket': u"Y's Mart"}
{u'Count': 3, u'City': u'bunkyo', u'Supermarket': u'SainE'}
{u'Count': 3, u'City': u'bunkyo', u'Supermarket': u'DAIMARU Peacock'}


{u'Count': 9, u'City': u'hoodi', u'Supermarket': u'Reliance Fresh'}
{u'Count': 9, u'City': u'hoodi'}
{u'Count': 6, u'City': u'hoodi', u'Supermarket': u"Nilgiri's"}
{u'Count': 3, u'City': u'hoodi', u'Supermarket': u'Royal Mart Supermarket'}
{u'Count': 3, u'City': u'hoodi', u'Supermarket': u'Safal'}



"""
Explanation: most popular supermarkets
End of explanation
"""


