from collections import defaultdict

import xml.etree.cElementTree as ET
import re

def get_element(osm_file, tags=('node', 'way', 'relation')):
    """Yield element if it is the right type of tag

    Reference:
    http://stackoverflow.com/questions/3095434/inserting-newlines-in-xml-file-generated-via-xml-etree-elementtree-in-python
    """
    context = iter(ET.iterparse(osm_file, events=('start', 'end')))
    _, root = next(context)
    for event, elem in context:
        if event == 'end' and elem.tag in tags:
            yield elem
            root.clear()

def take_sample(k, osm_file, sample_file):
    with open(sample_file, 'wb') as output:
        output.write('<?xml version="1.0" encoding="UTF-8"?>\n')
        output.write('<osm>\n  ')

        # Write every kth top level element
        for i, element in enumerate(get_element(osm_file)):
            if i % k == 0:
                # print "i is {}".format(i)
                output.write(ET.tostring(element, encoding='utf-8'))

        output.write('</osm>')

#take_sample(10, "../chicago_illinois.osm", "../sample_10.osm")

"""
Explanation: Introduction

We know how to get data now we focus on data itself
how good is it?
can it be trusted?

What is Data Cleaning

iterative process - detect, correct and loop
problems in data
text where we expect to find numeric data (two instead of 2)
may have missing elements, extra elements or totally different structure
out of range numbers
outliers compared to standard distributions
date entries in different formats



Sources of dirt data

user entry errors
poor data coding standard
different schemas for same kind of data
legacy systems
evolving systems
no unique identifiers
lost data during data migration
programmer error
corruption in transmission

Measures of data quality

Validity: Conforms to a schema or other constraints
Accuracy: Conforms to gold standards
Completeness: all records?
Consistency: matches other data
uniformity: same units

Blueprint for cleaning

Audit data 
programmatically check data and create a report on quality of data
statistical analysis to check for outliers


create a data cleaning plan
identify causes
define operations to clean
test to ensure operations will clean as expected


run the plan
possibly manually cleaning

Cleaning using blueprint

We will use a sub set of open street map data for Chicago to see cleaning
We will initially take a sample of our OSM file as the main one is quite big. The main file for Chicago can be downloaded here
End of explanation
"""


def print_sorted_dict(d):
    keys = d.keys()
    keys = sorted(keys, key=lambda s: s.lower())
    for k in keys:
        v = d[k]
        print "%s: %d" % (k, v) 

street_type_re = re.compile(r'\S+\.?$', re.IGNORECASE)
street_types = defaultdict(int)

def audit_street_type(street_types, street_name):
    m = street_type_re.search(street_name)
    if m:
        street_type = m.group()

        street_types[street_type] += 1

def is_street_name(elem):
    return (elem.tag == "tag") and (elem.attrib['k'] == "addr:street")

def audit1(osm_file):
    for event, elem in ET.iterparse(osm_file):
        if is_street_name(elem):
            audit_street_type(street_types, elem.attrib['v'])    
    print_sorted_dict(street_types)    

audit1("../sample_10.osm")

"""
Explanation: Now we will audit street types
End of explanation
"""


import csv

def get_row(input_file, skip_lines):
    rows = csv.DictReader(open(input_file))

    skipped = 0
    for row in rows:
        if skipped < skip_lines:
            skipped += 1
            continue
        
        yield row

def is_number(v):
    try:
        float(v)
        return True
    except:
        return False

def ensure_float(v):
    if is_number(v):
        return float(v)
    else:
        print "v is {}".format(v)

import math

def audit_population_denisty(input_file):
    for city in get_row(input_file, 3):
        population = ensure_float(city['populationTotal'])
        area = ensure_float(city['areaLand'])
        population_denisty = ensure_float(city['populationDensity'])
        if population and area and population_denisty:
            calculated_density = population / area
            
            if math.fabs(calculated_density - population_denisty) > 10:
                print "Possibly bad population denisty for {}".format(city['name'])

audit_population_denisty('cities.csv')

"""
Explanation: We have taken the street types that the parser recognized. 
we can see that there are multiple forms of each type. e.g. we have multiple forms of avenue.
now we have street types we would decide what type of cleaning do we want. Which ones do we keep and which to rename? Do we want to?

Auditing Validity
We may have different type of validity checks

mandatory
unique
foreign key
cross field constraints e.g. start date must be before end date
fields should have specific type e.g. numerical 2 vs. two
patterns
in a range
should be in a set

Wikipedia Infobox Dataset

wikipedia has data infoboxes besides main article
dbpedia has taken data from wikipedia and shared in a way that we can download
we will use cities data set that contains information about all cities

Auditing a Cross-Field Constrain
End of explanation
"""


import csv
import pprint

def process_file(input_file, output_good, output_bad):

    with open(input_file, "r") as f, open(output_good, 'w') as good, open(output_bad, 'w') as bad:
        reader = csv.DictReader(f)

        header = reader.fieldnames

        good_writer = csv.DictWriter(good, delimiter=",", fieldnames=header)
        good_writer.writeheader()
        bad_writer = csv.DictWriter(bad, delimiter=",", fieldnames=header)
        bad_writer.writeheader()

        for line in reader:
            uri = line['URI']
            if 'dbpedia.org' not in uri:
                continue

            production_start_year = line['productionStartYear']
            try:
                year = int(production_start_year[0:4])
                good_writer.writerow(line)
            except ValueError:
                bad_writer.writerow(line)

process_file('../autos.csv', 'autos-valid.csv', 'FIXME-autos.csv')

"""
Explanation: Correcting Validity
Your task is to check the "productionStartYear" of the DBPedia autos datafile for valid values.
The following things should be done:
- check if the field "productionStartYear" contains a year
- check if the year is in range 1886-2014
- convert the value of the field to be just a year (not full datetime)
- the rest of the fields and values should stay the same
- if the value of the field is a valid year in the range as described above,
  write that line to the output_good file
- if the value of the field is not a valid year as described above,
  write that line to the output_bad file
- discard rows (neither write to good nor bad) if the URI is not from dbpedia.org
- you should use the provided way of reading and writing data (DictReader and DictWriter)
  They will take care of dealing with the header.
End of explanation
"""


#counts is {'nulls': 0, 'empties': 0, 'arrays': 0}
def audit_float_field(v, counts, min_val, max_val):
    v = v.strip()
    if v == 'NULL':
        counts['nulls'] += 1
    elif v == "":
        counts['empties'] += 1
    elif is_array(v):
        counts['arrays'] += 1
    elif not is_number(v):
        print "found not number {}".format(v)
    else:
        v = float(v)
        if not (min_val <= v <= max_val):
            print "found out of range value {}".format(v)

"""
Explanation: Auditing Accuracy

difficult to do as needs gold standard data

Problems faced

some values are arrays
value of one column written in another column
underscores used - regex can be used to extract the actual data
possibly valid countries - in another language

Auditing Completeness

can be difficult to audit as "You don't know what you don't know"
about missing records
need reference data

Auditing Consistency

two different entries should not contradict each other
to fix this 
which data source do we trust more?
which data collection method is more reliable?



Auditing Uniformity

All fields using same units of measurement
pick the type of the field and see whether it is as per your expectation or not
print out unexpected things to see what is happening with your data
End of explanation
"""


import codecs
import csv
import json
import pprint

def audit_file(file_name, fields):
    field_types = {}
    for field in fields:
        field_types[field] = set()

    f = open(file_name, "r")
    reader = csv.DictReader(f)
    for line in reader:
        # print line
        for field in fields:
            field_value = line[field].strip()

            # if field == "name" or field == "areaLand":
            #     print "field {} has value {}".format(field, field_value)

            if field_value == "NULL" or field_value == "":
                field_types[field].add(type(None))
                continue

            if field_value.startswith("{"):
                field_types[field].add(type([]))

            try:
                int(field_value)
                field_types[field].add(type(1))
            except:
                try:
                    float(field_value)
                    field_types[field].add(type(1.1))
                except:
                    pass

    pprint.pprint(field_types)

    return field_types

FIELDS = ["name", "timeZone_label", "utcOffset", "homepage", "governmentType_label",
          "isPartOf_label", "areaCode", "populationTotal", "elevation",
          "maximumElevation", "minimumElevation", "populationDensity",
          "wgs84_pos#lat", "wgs84_pos#long", "areaLand", "areaMetro", "areaUrban"]

def test1(cities):
    field_types = audit_file(cities, FIELDS)

    assert field_types["areaLand"] == {type(1.1), type([]), type(None)}
    assert field_types['areaMetro'] == {type(1.1), type(None)}

test1('cities.csv')

"""
Explanation: More About Correcting Data

we are removing typographical errors or correcting them
validating against known list of entities or cross check against validated data set
data enhancement like N, E, W, S to North, East, West, South
changing reference data - 2 digit country code to 3 digit country code

PROBLEM SET START HERE
In this problem set you work with cities infobox data, audit it, come up with a
cleaning idea and then clean it up. In the first exercise we want you to audit
the datatypes that can be found in some particular fields in the dataset.
The possible types of values can be:
- NoneType if the value is a string "NULL" or an empty string ""
- list, if the value starts with "{"
- int, if the value can be cast to int
- float, if the value can be cast to float, but CANNOT be cast to int.
   For example, '3.23e+07' should be considered a float because it can be cast
   as float but int('3.23e+07') will throw a ValueError
- 'str', for all other values
The audit_file function should return a dictionary containing fieldnames and a 
SET of the types that can be found in the field. e.g.
{"field1": set([type(float()), type(int()), type(str())]),
 "field2": set([type(str())]),
  ....
}
The type() function returns a type object describing the argument given to the 
function. You can also use examples of objects to create type objects, e.g.
type(1.1) for a float: see the test function below for examples.
Note that the first three rows (after the header row) in the cities.csv file
are not actual data points. The contents of these rows should note be included
when processing data types. Be sure to include functionality in your code to
skip over or detect these rows.
End of explanation
"""


def fix_area(area):

    result = area

    if area == "NULL" or area == "":
        result = None
    elif area.startswith("{"):
        areas = area.strip("{}").split("|")
        # area =
        # area1 = float(areas[1])
        result = float(areas[0])

    print "for {} output is {}".format(area, result)

    return result

"""
Explanation: Since in the previous quiz you made a decision on which value to keep for the
"areaLand" field, you now know what has to be done.
Finish the function fix_area(). It will receive a string as an input, and it
has to return a float representing the value of the area or None.
You have to change the function fix_area. You can use extra functions if you
like, but changes to process_file will not be taken into account.
The rest of the code is just an example on how this function can be used.
End of explanation
"""


def fix_name(name):

    result = name
    if name.startswith("{"):
        result = name.strip("{}").split("|")
    else:
        result = [name]

    return result

def process_file_fun(filename, fun):
    data = []

    with open(filename, "r") as f:
        reader = csv.DictReader(f)

        #skipping the extra metadata
        for i in range(3):
            l = reader.next()

        # processing file
        for line in reader:
            fun(line)
            
            #pprint.pprint(line)
            
            data.append(line)

    return data

def fixArea(line):
    if "areaLand" in line:
        line["areaLand"] = fix_area(line["areaLand"])

def fixName(line):
    if "name" in line:
        line["name"] = fix_name(line["name"])

def test2(cities):
    data = process_file_fun(cities, fixArea)

    print "Printing three example results:"
    for n in range(5, 8):
        pprint.pprint(data[n]["areaLand"])
        
    pprint.pprint(data[3])

    assert data[3]["areaLand"] == None
    assert data[8]["areaLand"] == 55166700.0
    assert data[20]["areaLand"] == 14580000.0

test2('cities.csv')

def test3(cities):
    data = process_file_fun(cities, fixName)

    #print "Printing 20 results:"
    #for n in range(20):
    #    pprint.pprint(data[n]["name"])
    #pprint.pprint("data[14] is {}".format(data[14]))

    assert data[14]["name"] == ['Negtemiut', 'Nightmute']
    assert data[9]["name"] == ['Pell City Alabama']
    assert data[3]["name"] == ['Kumhari']

test3("cities.csv")

"""
Explanation: In the previous quiz you recognized that the "name" value can be an array (or
list in Python terms). It would make it easier to process and query the data
later if all values for the name are in a Python list, instead of being
just a string separated with special characters, like now.
Finish the function fix_name(). It will recieve a string as an input, and it
will return a list of all the names. If there is only one name, the list will
have only one item in it; if the name is "NULL", the list should be empty.
The rest of the code is just an example on how this function can be used.
End of explanation
"""


