'This will calculate the stats_demographic_aggregates for the
specified records. Either all (when rebuild_all is invoked) or for
the individual parameter(s) at the specified location(s) when run
onapprove - which currently happens inside the vulnerability
approve_report() controller.
@ToDo: onapprove/onaccept wrapper function for other workflows.
This will get the raw data from stats_demographic_data and generate
a stats_demographic_aggregate record for the given time period.
The reason for doing this is so that all aggregated data can be
obtained from a single table. So when displaying data for a
particular location it will not be necessary to try the aggregate
table, and if it\'s not there then try the data table. Rather just
look at the aggregate table.
Once this has run then a complete set of aggregate records should
exists for this parameter_id and location for every time period from
the first data item until the current time period.
@ToDo: Add test cases to modules/unit_tests/s3db/stats.py'

@staticmethod
def stats_demographic_update_aggregates(records=None):

    if (not records):
        return
    from dateutil.rrule import rrule, YEARLY
    db = current.db
    s3db = current.s3db
    dtable = s3db.stats_demographic_data
    atable = db.stats_demographic_aggregate
    gtable = db.gis_location
    param_total_dict = {}
    param_location_dict = {}
    location_dict = {}
    loc_level_list = {}
    aggregated_period = S3StatsDemographicModel.stats_demographic_aggregated_period
    (last_period, year_end) = aggregated_period(None)
    if isinstance(records, basestring):
        from_json = True
        from dateutil.parser import parse
        records = json.loads(records)
    elif isinstance(records[0]['stats_demographic_data']['date'], (datetime.date, datetime.datetime)):
        from_json = False
    else:
        from_json = True
        from dateutil.parser import parse
    for record in records:
        total_id = record['stats_demographic']['total_id']
        record = record['stats_demographic_data']
        data_id = record['data_id']
        location_id = record['location_id']
        parameter_id = record['parameter_id']
        if ((not location_id) or (not parameter_id)):
            current.log.warning(('Skipping    bad    stats_demographic_data    record    with    data_id    %s    ' % data_id))
            continue
        if (total_id and (parameter_id not in param_total_dict)):
            param_total_dict[parameter_id] = total_id
        if from_json:
            date = parse(record['date'])
            date = date.date()
        else:
            date = record['date']
        (start_date, end_date) = aggregated_period(date)
        query = (((dtable.location_id == location_id) & (dtable.deleted != True)) & (dtable.approved_by != None))
        fields = [dtable.data_id, dtable.date, dtable.value]
        if total_id:
            query &= dtable.parameter_id.belongs([parameter_id, total_id])
            fields.append(dtable.parameter_id)
        else:
            percentage = None
            query &= (dtable.parameter_id == parameter_id)
        data_rows = db(query).select(*fields)
        if total_id:
            total_rows = data_rows.exclude((lambda row: (row.parameter_id == total_id)))
            earliest_period = current.request.utcnow.date()
            end_date = year_end
            totals = {}
            for row in total_rows:
                row_date = row.date
                (start_date, end_date) = aggregated_period(row_date)
                if (start_date in totals):
                    if (row_date <= totals[start_date]['date']):
                        continue
                elif (start_date < earliest_period):
                    earliest_period = start_date
                totals[start_date] = Storage(date=row_date, id=row.data_id, value=row.value)
        earliest_period = start_date
        end_date = year_end
        data = {}
        data[start_date] = Storage(date=date, id=data_id, value=record['value'])
        for row in data_rows:
            if (row.data_id == data_id):
                continue
            row_date = row.date
            (start_date, end_date) = aggregated_period(row_date)
            if (start_date in data):
                if (row_date <= data[start_date]['date']):
                    continue
            elif (start_date < earliest_period):
                earliest_period = start_date
            data[start_date] = Storage(date=row_date, id=row.data_id, value=row.value)
        query = ((atable.location_id == location_id) & (atable.parameter_id == parameter_id))
        aggr_rows = db(query).select(atable.id, atable.agg_type, atable.date, atable.end_date, atable.sum)
        aggr = {}
        for row in aggr_rows:
            (start_date, end_date) = aggregated_period(row.date)
            aggr[start_date] = Storage(id=row.id, type=row.agg_type, end_date=row.end_date, sum=row.sum)
        last_data_period = earliest_period
        last_type_agg = False
        last_data_value = None
        last_total = None
        changed_periods = []
        for dt in rrule(YEARLY, dtstart=earliest_period, until=last_period):
            dt = dt.date()
            if (dt != last_period):
                (start_date, end_date) = aggregated_period(dt)
            else:
                start_date = dt
                end_date = None
            if (dt in aggr):
                agg_type = aggr[dt]['type']
                if (agg_type == 2):
                    last_type_agg = True
                    last_data_value = aggr[dt]['sum']
                    continue
                query = (atable.id == aggr[dt]['id'])
                if (agg_type == 3):
                    if (dt in data):
                        value = data[dt]['value']
                        last_data_value = value
                        if total_id:
                            if (dt in totals):
                                last_total = totals[dt]['value']
                            if last_total:
                                percentage = ((100 * value) / last_total)
                                percentage = round(percentage, 3)
                        db(query).update(agg_type=1, end_date=end_date, percentage=percentage, sum=value)
                        changed_periods.append((start_date, end_date))
                    elif last_type_agg:
                        continue
                    elif (aggr[dt]['sum'] != last_data_value):
                        value = last_data_value
                        if total_id:
                            if (dt in totals):
                                last_total = totals[dt]['value']
                            if last_total:
                                percentage = ((100 * value) / last_total)
                                percentage = round(percentage, 3)
                        db(query).update(agg_type=3, end_date=end_date, percentage=percentage, sum=value)
                        changed_periods.append((start_date, end_date))
                elif (agg_type == 1):
                    if (dt in data):
                        value = data[dt]['value']
                        last_data_value = value
                        if (total_id and (dt in totals)):
                            last_total = totals[dt]['value']
                        if (aggr[dt]['sum'] != value):
                            if (total_id and last_total):
                                percentage = ((100 * value) / last_total)
                                percentage = round(percentage, 3)
                            db(query).update(agg_type=1, end_date=end_date, percentage=percentage, sum=value)
                            changed_periods.append((start_date, end_date))
                    else:
                        value = last_data_value
                        if total_id:
                            if (dt in totals):
                                last_total = totals[dt]['value']
                            if last_total:
                                percentage = ((100 * value) / last_total)
                                percentage = round(percentage, 3)
                        db(query).update(agg_type=3, end_date=end_date, percentage=percentage, sum=value)
                        changed_periods.append((start_date, end_date))
            else:
                if (dt in data):
                    value = data[dt]['value']
                    agg_type = 1
                    last_data_value = value
                else:
                    value = last_data_value
                    agg_type = 3
                if total_id:
                    if (dt in totals):
                        last_total = totals[dt]['value']
                    if last_total:
                        percentage = ((100 * value) / last_total)
                        percentage = round(percentage, 3)
                atable.insert(parameter_id=parameter_id, location_id=location_id, agg_type=agg_type, date=start_date, end_date=end_date, percentage=percentage, sum=value)
                changed_periods.append((start_date, end_date))
        if (changed_periods == []):
            continue
        location = db((gtable.id == location_id)).select(gtable.level, limitby=(0, 1)).first()
        loc_level_list[location_id] = location.level
        if (parameter_id not in param_location_dict):
            param_location_dict[parameter_id] = {location_id: changed_periods}
        elif (location_id not in param_location_dict[parameter_id]):
            param_location_dict[parameter_id][location_id] = changed_periods
        elif (changed_periods[0][0] < param_location_dict[parameter_id][location_id][0][0]):
            param_location_dict[parameter_id][location_id] = changed_periods
        if (location_id not in location_dict):
            location_dict[location_id] = changed_periods
        elif (changed_periods[0][0] < location_dict[location_id][0][0]):
            location_dict[location_id] = changed_periods
    parents = {}
    get_parents = current.gis.get_parents
    for loc_id in location_dict.keys():
        _parents = get_parents(loc_id)
        if parents:
            parents[loc_id] = _parents
    parents_data = {}
    for (param_id, loc_dict) in param_location_dict.items():
        for (loc_id, periods) in loc_dict.items():
            if (loc_id in parents):
                for p_loc_row in parents[loc_id]:
                    p_loc_id = p_loc_row.id
                    if (param_id in parents_data):
                        if (p_loc_id in parents_data[param_id]):
                            if (periods[0][0] < parents_data[param_id][p_loc_id][0][0][0]):
                                parents_data[param_id][p_loc_id][0] = periods
                        else:
                            parents_data[param_id][p_loc_id] = [periods, loc_level_list[loc_id]]
                    else:
                        parents_data[param_id] = {p_loc_id: [periods, loc_level_list[loc_id]]}
    async = current.s3task.async
    for (param_id, loc_dict) in parents_data.items():
        total_id = param_total_dict[param_id]
        for (loc_id, (changed_periods, loc_level)) in loc_dict.items():
            for (start_date, end_date) in changed_periods:
                (s, e) = (str(start_date), str(end_date))
                async('stats_demographic_update_aggregate_location', args=[loc_level, loc_id, param_id, total_id, s, e], timeout=1800)
