timeseries_scripts/make_json.py
def make_json(data_sets, info_cols, version, changes, headers, areas):
    '''
    Create a datapackage.json file that complies with the Frictionless
    data JSON Table Schema from the information in the column-MultiIndex.

    Parameters
    ----------
    data_sets: dict of pandas.DataFrames
        A dict with keys '15min' and '60min' and values the respective
        DataFrames
    info_cols : dict of strings
        Names for non-data columns such as for the index, for additional 
        timestamps or the marker column
    version: str
        Version tag of the Data Package
    changes : str
        Desription of the changes from the last version to this one.
    headers : list
        List of strings indicating the level names of the pandas.MultiIndex
        for the columns of the dataframe.

    Returns
    ----------
    None

    '''

    # list of files included in the datapackage in YAML-format
    resource_list = '''
- mediatype: application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
  format: xlsx
  path: time_series.xlsx
'''
    source_list = ''  # list of sources were data comes from in YAML-format
    schemas_dict = ''  # dictionary of schemas in YAML-format

    for res_key, df in data_sets.items():
        field_list = ''  # list of columns in a file in YAML-format

        # Both datasets (15min and 60min) get an antry in the resource list
        resource_list = resource_list + resource_template.format(
            res_key=res_key)

        # Create the list of of columns in a file, starting with the index
        # field
        for col in df.columns:
            if col[0] in info_cols.values():
                continue
            h = {k: v for k, v in zip(headers, col)}
            row = areas['OPSD area'] == h['region']
            primary_concept = areas.loc[row, 'primary concept'].values[0]
            geo = areas[primary_concept][row].values[0]
            if not primary_concept == 'country':
                geo = geo + ' (' + primary_concept + ')'

            descriptions = yaml.load(
                descriptions_template.format(
                    tech=h['variable'], unit=h['unit'], geo=geo))
            try:
                h['description'] = descriptions[h['attribute']]
            except KeyError:
                h['description'] = descriptions[h['variable']]
            field_list = field_list + field_template.format(**h)
            source_list = source_list + source_template.format(**h)
        schemas_dict = schemas_dict + schemas_template.format(
            res_key=res_key, **info_cols) + field_list

    # Remove duplicates from sources_list. set() returns unique values from a
    # collection, but it cannot compare dicts. Since source_list is a list of of
    # dicts, this requires some juggling with data types
    source_list = [dict(tupleized)
                   for tupleized in set(tuple(entry.items())
                                        for entry in yaml.load(source_list))]

    # Parse the YAML-Strings and stitch the building blocks together
    metadata = yaml.load(metadata_head.format(
        version=version, changes=changes))
    metadata['sources'] = source_list
    metadata['resources'] = yaml.load(resource_list)
    metadata['schemas'] = yaml.load(schemas_dict)

    # Remove URL for source if a column is based on own calculations
    for schema in metadata['schemas'].values():
        for field in schema['fields']:
            if ('source' in field.keys() and
                    field['source']['name'] == 'own calculation'):
                del field['source']['web']

    # write the metadata to disk
    datapackage_json = json.dumps(metadata, indent=4, separators=(',', ': '))
    with open('datapackage.json', 'w') as f:
        f.write(datapackage_json)

    return

    Create a datapackage.json file that complies with the Frictionless
    data JSON Table Schema from the information in the column-MultiIndex.

    Parameters
    ----------
    data_sets: dict of pandas.DataFrames
        A dict with keys '15min' and '60min' and values the respective
        DataFrames
    info_cols : dict of strings
        Names for non-data columns such as for the index, for additional 
        timestamps or the marker column
    version: str
        Version tag of the Data Package
    changes : str
        Desription of the changes from the last version to this one.
    headers : list
        List of strings indicating the level names of the pandas.MultiIndex
        for the columns of the dataframe.

    Returns
    ----------
    None

    
Create a datapackage.json file that complies with the Frictionless
data JSON Table Schema from the information in the column-MultiIndex.

Parameters

dict of pandas.DataFrames
A dict with keys '15min' and '60min' and values the respective
DataFrames
info_cols : dict of strings
Names for non-data columns such as for the index, for additional
timestamps or the marker column
version: str
Version tag of the Data Package
changes : str
Desription of the changes from the last version to this one.
headers : list
List of strings indicating the level names of the pandas.MultiIndex
for the columns of the dataframe.

Returns

None
Create a datapackage . json file that complies with the Frictionless data JSON Table Schema from the information in the column - MultiIndex . Parameters dict of pandas . DataFrames A dict with keys ' 15min ' and ' 60min ' and values the respective DataFrames info_cols : dict of strings Names for non - data columns such as for the index for additional timestamps or the marker column version : str Version tag of the Data Package changes : str Desription of the changes from the last version to this one . headers : list List of strings indicating the level names of the pandas . MultiIndex for the columns of the dataframe . Returns None
def make_json(data_sets, info_cols, version, changes, headers, areas):
    resource_list = '''
- mediatype: application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
  format: xlsx
  path: time_series.xlsx
'''
    source_list = ''  
    schemas_dict = ''  
    for res_key, df in data_sets.items():
        field_list = ''  
        resource_list = resource_list + resource_template.format(
            res_key=res_key)
        for col in df.columns:
            if col[0] in info_cols.values():
                continue
            h = {k: v for k, v in zip(headers, col)}
            row = areas['OPSD area'] == h['region']
            primary_concept = areas.loc[row, 'primary concept'].values[0]
            geo = areas[primary_concept][row].values[0]
            if not primary_concept == 'country':
                geo = geo + ' (' + primary_concept + ')'
            descriptions = yaml.load(
                descriptions_template.format(
                    tech=h['variable'], unit=h['unit'], geo=geo))
            try:
                h['description'] = descriptions[h['attribute']]
            except KeyError:
                h['description'] = descriptions[h['variable']]
            field_list = field_list + field_template.format(**h)
            source_list = source_list + source_template.format(**h)
        schemas_dict = schemas_dict + schemas_template.format(
            res_key=res_key, **info_cols) + field_list
    source_list = [dict(tupleized)
                   for tupleized in set(tuple(entry.items())
                                        for entry in yaml.load(source_list))]
    metadata = yaml.load(metadata_head.format(
        version=version, changes=changes))
    metadata['sources'] = source_list
    metadata['resources'] = yaml.load(resource_list)
    metadata['schemas'] = yaml.load(schemas_dict)
    for schema in metadata['schemas'].values():
        for field in schema['fields']:
            if ('source' in field.keys() and
                    field['source']['name'] == 'own calculation'):
                del field['source']['web']
    datapackage_json = json.dumps(metadata, indent=4, separators=(',', ': '))
    with open('datapackage.json', 'w') as f:
        f.write(datapackage_json)
    return
def make_json ( data_sets , info_cols , version , changes , headers , areas ) : resource_list = '''
- mediatype: application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
  format: xlsx
  path: time_series.xlsx
''' source_list = '' schemas_dict = '' for res_key , df in data_sets . items ( ) : field_list = '' resource_list = resource_list + resource_template . format ( res_key = res_key ) for col in df . columns : if col [ 0 ] in info_cols . values ( ) : continue h = { k : v for k , v in zip ( headers , col ) } row = areas [ 'OPSD area' ] == h [ 'region' ] primary_concept = areas . loc [ row , 'primary concept' ] . values [ 0 ] geo = areas [ primary_concept ] [ row ] . values [ 0 ] if not primary_concept == 'country' : geo = geo + ' (' + primary_concept + ')' descriptions = yaml . load ( descriptions_template . format ( tech = h [ 'variable' ] , unit = h [ 'unit' ] , geo = geo ) ) try : h [ 'description' ] = descriptions [ h [ 'attribute' ] ] except KeyError : h [ 'description' ] = descriptions [ h [ 'variable' ] ] field_list = field_list + field_template . format ( ** h ) source_list = source_list + source_template . format ( ** h ) schemas_dict = schemas_dict + schemas_template . format ( res_key = res_key , ** info_cols ) + field_list source_list = [ dict ( tupleized ) for tupleized in set ( tuple ( entry . items ( ) ) for entry in yaml . load ( source_list ) ) ] metadata = yaml . load ( metadata_head . format ( version = version , changes = changes ) ) metadata [ 'sources' ] = source_list metadata [ 'resources' ] = yaml . load ( resource_list ) metadata [ 'schemas' ] = yaml . load ( schemas_dict ) for schema in metadata [ 'schemas' ] . values ( ) : for field in schema [ 'fields' ] : if ( 'source' in field . keys ( ) and field [ 'source' ] [ 'name' ] == 'own calculation' ) : del field [ 'source' ] [ 'web' ] datapackage_json = json . dumps ( metadata , indent = 4 , separators = ( ',' , ': ' ) ) with open ( 'datapackage.json' , 'w' ) as f : f . write ( datapackage_json ) return
Create a datapackage.json file that complies with the Frictionless
data JSON Table Schema from the information in the column-MultiIndex.
Create a datapackage . json file that complies with the Frictionless data JSON Table Schema from the information in the column - MultiIndex .
'''
    Create a datapackage.json file that complies with the Frictionless
    data JSON Table Schema from the information in the column-MultiIndex.

    Parameters
    ----------
    data_sets: dict of pandas.DataFrames
        A dict with keys '15min' and '60min' and values the respective
        DataFrames
    info_cols : dict of strings
        Names for non-data columns such as for the index, for additional 
        timestamps or the marker column
    version: str
        Version tag of the Data Package
    changes : str
        Desription of the changes from the last version to this one.
    headers : list
        List of strings indicating the level names of the pandas.MultiIndex
        for the columns of the dataframe.

    Returns
    ----------
    None

    ''' # list of files included in the datapackage in YAML-format # list of sources were data comes from in YAML-format # dictionary of schemas in YAML-format # list of columns in a file in YAML-format # Both datasets (15min and 60min) get an antry in the resource list # Create the list of of columns in a file, starting with the index # field # Remove duplicates from sources_list. set() returns unique values from a # collection, but it cannot compare dicts. Since source_list is a list of of # dicts, this requires some juggling with data types # Parse the YAML-Strings and stitch the building blocks together # Remove URL for source if a column is based on own calculations # write the metadata to disk
data_sets
info_cols
version
changes
headers
areas