backend/model_api/migrations/0009_load_foregn_predictions.py
def load_csv(apps, url, state_mapping):
    """
    Reads the given csv url and <state id, state name> mapping,
    returns a list of Covid19PredictionDataPoint objects.
    Only the date, area, and val fields on the objects are set. Note
    that the objects have not been saved into the database yet.
    :param apps: Django apps object.
    :param path: Path to CSV with prediction data.
    :return: List of Covid19PredictionDataPoint objects (NOT SAVED YET).
    """
    Area = apps.get_model('model_api', 'Area')
    Covid19PredictionDataPoint = apps.get_model(
        'model_api', 'Covid19PredictionDataPoint')

    try:
        f = io.StringIO(urllib.request.urlopen(url).read().decode('utf-8'))
        reader = csv.reader(f)
        header = next(reader, None)

        # Because different csv files have different column arrangements,
        # find out the index the columns containing different data fields first.
        location_col = -1
        date_col = -1
        target_col = -1
        type_col = -1
        value_col = -1

        for i in range(0, len(header)):
            if (header[i] == "location"):
                location_col = i
            elif (header[i] == "target_end_date"):
                date_col = i
            elif (header[i] == "target"):
                target_col = i
            elif (header[i] == "type"):
                type_col = i
            elif (header[i] == "value"):
                value_col = i

        data = []

        for row in reader:
            # Skip the row of quantile-type prediction or not cumulative type.
            if (row[type_col] != "point"  or "cum death" not in row[target_col]):
                continue

            area = None
            state = ""
            country = ""

            if row[location_col] == "US":
                country = "US"
            else:
                country = "US"
                state_id = int(row[location_col])
                state = state_mapping[state_id]

            # Try to find the corresponding area.
            try:
                area = Area.objects.get(country=country, state=state)
            except Area.DoesNotExist:
                msg = "Could not find the area for country '{0}'".format(
                    country)
                if state:
                    msg += " and state '{0}'".format(state)
                msg += ' in model_api_area. Skip this area.'
                print(msg)
                continue

            except Area.MultipleObjectsReturned:
                msg = "Found multiple areas for country '{0}'".format(
                    country)
                if state:
                    msg += " and state '{0}'".format(state)
                msg += ' in model_api_area. Skip this area.'
                print(msg)
                continue

            raw_date = row[date_col]
            date = datetime.datetime(*[int(item) for item in raw_date.split('-')])

            # Skip invalid values.
            raw_val = row[value_col]
            if raw_val in ['NaN', '-Inf', 'Inf']:
                continue

            # Skip negative values.
            val = int(float(raw_val))
            if val < 0:
                continue

            data.append(Covid19PredictionDataPoint(
                area=area,
                date=date,
                val=val
            ))

        return data

    except urllib.error.HTTPError as httpe:
        print("A HttpError is found when loading data from" + url)
        return []
    except urllib.error.URLError as urle:
        print("A URLError is found when loading data from" + url)
        return []

    Reads the given csv url and <state id, state name> mapping,
    returns a list of Covid19PredictionDataPoint objects.
    Only the date, area, and val fields on the objects are set. Note
    that the objects have not been saved into the database yet.
    :param apps: Django apps object.
    :param path: Path to CSV with prediction data.
    :return: List of Covid19PredictionDataPoint objects (NOT SAVED YET).
    
Reads the given csv url and  mapping,
returns a list of Covid19PredictionDataPoint objects.
Only the date, area, and val fields on the objects are set. Note
that the objects have not been saved into the database yet.
Reads the given csv url and mapping returns a list of Covid19PredictionDataPoint objects . Only the date area and val fields on the objects are set . Note that the objects have not been saved into the database yet .
def load_csv(apps, url, state_mapping):
    Area = apps.get_model('model_api', 'Area')
    Covid19PredictionDataPoint = apps.get_model(
        'model_api', 'Covid19PredictionDataPoint')
    try:
        f = io.StringIO(urllib.request.urlopen(url).read().decode('utf-8'))
        reader = csv.reader(f)
        header = next(reader, None)
        location_col = -1
        date_col = -1
        target_col = -1
        type_col = -1
        value_col = -1
        for i in range(0, len(header)):
            if (header[i] == "location"):
                location_col = i
            elif (header[i] == "target_end_date"):
                date_col = i
            elif (header[i] == "target"):
                target_col = i
            elif (header[i] == "type"):
                type_col = i
            elif (header[i] == "value"):
                value_col = i
        data = []
        for row in reader:
            if (row[type_col] != "point"  or "cum death" not in row[target_col]):
                continue
            area = None
            state = ""
            country = ""
            if row[location_col] == "US":
                country = "US"
            else:
                country = "US"
                state_id = int(row[location_col])
                state = state_mapping[state_id]
            try:
                area = Area.objects.get(country=country, state=state)
            except Area.DoesNotExist:
                msg = "Could not find the area for country '{0}'".format(
                    country)
                if state:
                    msg += " and state '{0}'".format(state)
                msg += ' in model_api_area. Skip this area.'
                print(msg)
                continue
            except Area.MultipleObjectsReturned:
                msg = "Found multiple areas for country '{0}'".format(
                    country)
                if state:
                    msg += " and state '{0}'".format(state)
                msg += ' in model_api_area. Skip this area.'
                print(msg)
                continue
            raw_date = row[date_col]
            date = datetime.datetime(*[int(item) for item in raw_date.split('-')])
            raw_val = row[value_col]
            if raw_val in ['NaN', '-Inf', 'Inf']:
                continue
            val = int(float(raw_val))
            if val < 0:
                continue
            data.append(Covid19PredictionDataPoint(
                area=area,
                date=date,
                val=val
            ))
        return data
    except urllib.error.HTTPError as httpe:
        print("A HttpError is found when loading data from" + url)
        return []
    except urllib.error.URLError as urle:
        print("A URLError is found when loading data from" + url)
        return []
def load_csv ( apps , url , state_mapping ) : Area = apps . get_model ( 'model_api' , 'Area' ) Covid19PredictionDataPoint = apps . get_model ( 'model_api' , 'Covid19PredictionDataPoint' ) try : f = io . StringIO ( urllib . request . urlopen ( url ) . read ( ) . decode ( 'utf-8' ) ) reader = csv . reader ( f ) header = next ( reader , None ) location_col = - 1 date_col = - 1 target_col = - 1 type_col = - 1 value_col = - 1 for i in range ( 0 , len ( header ) ) : if ( header [ i ] == "location" ) : location_col = i elif ( header [ i ] == "target_end_date" ) : date_col = i elif ( header [ i ] == "target" ) : target_col = i elif ( header [ i ] == "type" ) : type_col = i elif ( header [ i ] == "value" ) : value_col = i data = [ ] for row in reader : if ( row [ type_col ] != "point" or "cum death" not in row [ target_col ] ) : continue area = None state = "" country = "" if row [ location_col ] == "US" : country = "US" else : country = "US" state_id = int ( row [ location_col ] ) state = state_mapping [ state_id ] try : area = Area . objects . get ( country = country , state = state ) except Area . DoesNotExist : msg = "Could not find the area for country '{0}'" . format ( country ) if state : msg += " and state '{0}'" . format ( state ) msg += ' in model_api_area. Skip this area.' print ( msg ) continue except Area . MultipleObjectsReturned : msg = "Found multiple areas for country '{0}'" . format ( country ) if state : msg += " and state '{0}'" . format ( state ) msg += ' in model_api_area. Skip this area.' print ( msg ) continue raw_date = row [ date_col ] date = datetime . datetime ( * [ int ( item ) for item in raw_date . split ( '-' ) ] ) raw_val = row [ value_col ] if raw_val in [ 'NaN' , '-Inf' , 'Inf' ] : continue val = int ( float ( raw_val ) ) if val < 0 : continue data . append ( Covid19PredictionDataPoint ( area = area , date = date , val = val ) ) return data except urllib . error . HTTPError as httpe : print ( "A HttpError is found when loading data from" + url ) return [ ] except urllib . error . URLError as urle : print ( "A URLError is found when loading data from" + url ) return [ ]
Reads the given csv url and <state id, state name> mapping,
returns a list of Covid19PredictionDataPoint objects.
Reads the given csv url and <state id state name > mapping returns a list of Covid19PredictionDataPoint objects .
"""
    Reads the given csv url and <state id, state name> mapping,
    returns a list of Covid19PredictionDataPoint objects.
    Only the date, area, and val fields on the objects are set. Note
    that the objects have not been saved into the database yet.
    :param apps: Django apps object.
    :param path: Path to CSV with prediction data.
    :return: List of Covid19PredictionDataPoint objects (NOT SAVED YET).
    """ # Because different csv files have different column arrangements, # find out the index the columns containing different data fields first. # Skip the row of quantile-type prediction or not cumulative type. # Try to find the corresponding area. # Skip invalid values. # Skip negative values.
List of Covid19PredictionDataPoint objects (NOT SAVED YET).
List of Covid19PredictionDataPoint objects ( NOT SAVED YET ) .
apps
Django apps object.
Django apps object .
url
state_mapping
path
Path to CSV with prediction data.
Path to CSV with prediction data .