harvester_odt/comunweb/converter.py
def comunweb_normalize_field_values(obj):
    """
    Normalize values from a ComunWeb object.

    :param obj:
        A metadata object in the format returned by ComunWeb.
        Some examples can be found in ``docs/open_data-node-*.json``.

        Basically, it is a dict with a top-level key "fields" pointing
        to a dict whose keys are mapping to dicts describing the field.

        Example::

            "fields": {
                "abstract": {
                    "classattribute_id": 2643,
                    "description": "",
                    "id": 7657698,
                    "identifier": "abstract",
                    "name": "Abstract (descrizione breve)",
                    "string_value": "Personale in servizio al 31/12/13. ...",
                    "type": "ezxmltext",
                    "value": "<p>Personale in servizio al 31/12/13.</p> ..."
                },
            }

    :return: a dict like this::

        {
            'field_name': {
                'label': 'Field label',
                'value': 'Field value',
            }
        }
    """

    def _get_field_value(fld):
        if fld['type'] == 'ezxmltext':
            # We don't want HTML entities here, instead we restore
            # original characters and let ckan do the escaping for us.

            # Otherwise, we will end up with a lot of ``[HTML
            # Removed]`` garbage inside our texts..

            return _html_unescape(fld['string_value']) or None

        if fld['type'] == 'ezstring':
            if fld['value'] != fld['string_value']:
                logger.warning(
                    "String fields value doesn't match string_value")
                logger.debug(
                    "value={0!r} string_value={1!r}".format(
                        fld['value'], fld['string_value']))
            return fld['value']  # should be == string_value

        if fld['type'] == 'ezkeyword':
            # This is a "tags" field
            # todo: should tags be splitted somehow?
            return fld['value'] or None

        if fld['type'] == 'ezdate':
            val = int(fld['value'])
            if val == 0:
                return None  # 0 is used as empty value

            # Note: we have no idea of which timezone has been used
            # to generate those timestamps -- let's assume UTC.

            # Otherwise, we should use pytz to read timestamp
            # and export it to the desired timezone.

            return datetime.datetime.utcfromtimestamp(val)

        if fld['type'] in ('ezbinaryfile', 'ezimage', 'ezurl'):
            # This is a URL pointing to something..
            return fld['value'] or None

        if fld['type'] == 'ezobjectrelationlist':
            # This is a relationship to some object
            # Value is something like this:

            # {
            # "classIdentifier": "tipo_formato",
            # "dateModified": 1362067615,
            # "datePublished": 1354868182,
            # "link": "http://.../api/opendata/v1/content/object/715852",
            # "objectId": 715852,
            # "objectName": "CSV (comma-separated values)",
            # "objectRemoteId": "356f4d6e6a1dd8d72ecfa2d12b061fcb"
            # }
            return None  # Not handled (yet)

        logger.warning('Encountered an unsupported field type: {0}'
                       .format(fld['type']))
        logger.debug('Field definition was: {0!r}'.format(fld))

    data = {}
    for fldname, fld in obj['fields'].iteritems():
        assert fld['identifier'] == fldname
        value = _get_field_value(fld)
        data[fldname] = {'value': value, 'label': fld['name']}
    return data

    Normalize values from a ComunWeb object.

    :param obj:
        A metadata object in the format returned by ComunWeb.
        Some examples can be found in ``docs/open_data-node-*.json``.

        Basically, it is a dict with a top-level key "fields" pointing
        to a dict whose keys are mapping to dicts describing the field.

        Example::

            "fields": {
                "abstract": {
                    "classattribute_id": 2643,
                    "description": "",
                    "id": 7657698,
                    "identifier": "abstract",
                    "name": "Abstract (descrizione breve)",
                    "string_value": "Personale in servizio al 31/12/13. ...",
                    "type": "ezxmltext",
                    "value": "<p>Personale in servizio al 31/12/13.</p> ..."
                },
            }

    :return: a dict like this::

        {
            'field_name': {
                'label': 'Field label',
                'value': 'Field value',
            }
        }
    
Normalize values from a ComunWeb object.
Normalize values from a ComunWeb object .
def comunweb_normalize_field_values(obj):
    def _get_field_value(fld):
        if fld['type'] == 'ezxmltext':
            return _html_unescape(fld['string_value']) or None
        if fld['type'] == 'ezstring':
            if fld['value'] != fld['string_value']:
                logger.warning(
                    "String fields value doesn't match string_value")
                logger.debug(
                    "value={0!r} string_value={1!r}".format(
                        fld['value'], fld['string_value']))
            return fld['value']  
        if fld['type'] == 'ezkeyword':
            return fld['value'] or None
        if fld['type'] == 'ezdate':
            val = int(fld['value'])
            if val == 0:
                return None  
            return datetime.datetime.utcfromtimestamp(val)
        if fld['type'] in ('ezbinaryfile', 'ezimage', 'ezurl'):
            return fld['value'] or None
        if fld['type'] == 'ezobjectrelationlist':
            return None  
        logger.warning('Encountered an unsupported field type: {0}'
                       .format(fld['type']))
        logger.debug('Field definition was: {0!r}'.format(fld))
    data = {}
    for fldname, fld in obj['fields'].iteritems():
        assert fld['identifier'] == fldname
        value = _get_field_value(fld)
        data[fldname] = {'value': value, 'label': fld['name']}
    return data
def comunweb_normalize_field_values ( obj ) : def _get_field_value ( fld ) : if fld [ 'type' ] == 'ezxmltext' : return _html_unescape ( fld [ 'string_value' ] ) or None if fld [ 'type' ] == 'ezstring' : if fld [ 'value' ] != fld [ 'string_value' ] : logger . warning ( "String fields value doesn't match string_value" ) logger . debug ( "value={0!r} string_value={1!r}" . format ( fld [ 'value' ] , fld [ 'string_value' ] ) ) return fld [ 'value' ] if fld [ 'type' ] == 'ezkeyword' : return fld [ 'value' ] or None if fld [ 'type' ] == 'ezdate' : val = int ( fld [ 'value' ] ) if val == 0 : return None return datetime . datetime . utcfromtimestamp ( val ) if fld [ 'type' ] in ( 'ezbinaryfile' , 'ezimage' , 'ezurl' ) : return fld [ 'value' ] or None if fld [ 'type' ] == 'ezobjectrelationlist' : return None logger . warning ( 'Encountered an unsupported field type: {0}' . format ( fld [ 'type' ] ) ) logger . debug ( 'Field definition was: {0!r}' . format ( fld ) ) data = { } for fldname , fld in obj [ 'fields' ] . iteritems ( ) : assert fld [ 'identifier' ] == fldname value = _get_field_value ( fld ) data [ fldname ] = { 'value' : value , 'label' : fld [ 'name' ] } return data
Normalize values from a ComunWeb object.
Normalize values from a ComunWeb object .
"""
    Normalize values from a ComunWeb object.

    :param obj:
        A metadata object in the format returned by ComunWeb.
        Some examples can be found in ``docs/open_data-node-*.json``.

        Basically, it is a dict with a top-level key "fields" pointing
        to a dict whose keys are mapping to dicts describing the field.

        Example::

            "fields": {
                "abstract": {
                    "classattribute_id": 2643,
                    "description": "",
                    "id": 7657698,
                    "identifier": "abstract",
                    "name": "Abstract (descrizione breve)",
                    "string_value": "Personale in servizio al 31/12/13. ...",
                    "type": "ezxmltext",
                    "value": "<p>Personale in servizio al 31/12/13.</p> ..."
                },
            }

    :return: a dict like this::

        {
            'field_name': {
                'label': 'Field label',
                'value': 'Field value',
            }
        }
    """ # We don't want HTML entities here, instead we restore # original characters and let ckan do the escaping for us. # Otherwise, we will end up with a lot of ``[HTML # Removed]`` garbage inside our texts.. # should be == string_value # This is a "tags" field # todo: should tags be splitted somehow? # 0 is used as empty value # Note: we have no idea of which timezone has been used # to generate those timestamps -- let's assume UTC. # Otherwise, we should use pytz to read timestamp # and export it to the desired timezone. # This is a URL pointing to something.. # This is a relationship to some object # Value is something like this: # { # "classIdentifier": "tipo_formato", # "dateModified": 1362067615, # "datePublished": 1354868182, # "link": "http://.../api/opendata/v1/content/object/715852", # "objectId": 715852, # "objectName": "CSV (comma-separated values)", # "objectRemoteId": "356f4d6e6a1dd8d72ecfa2d12b061fcb" # } # Not handled (yet)
a dict like this::
{
'field_name': {
'label': 'Field label',
'value': 'Field value',
}
}
a dict like this :: { ' field_name ' : { ' label ' : ' Field label ' ' value ' : ' Field value ' } }
obj
A metadata object in the format returned by ComunWeb.
Some examples can be found in ``docs/open_data-node-*.json``.

Basically, it is a dict with a top-level key "fields" pointing
to a dict whose keys are mapping to dicts describing the field.



"fields": {
"abstract": {
"classattribute_id": 2643,
"description": "",
"id": 7657698,
"identifier": "abstract",
"name": "Abstract (descrizione breve)",
"string_value": "Personale in servizio al 31/12/13.
A metadata object in the format returned by ComunWeb . Some examples can be found in ` ` docs / open_data - node - * . json ` ` . Basically it is a dict with a top - level key " fields " pointing to a dict whose keys are mapping to dicts describing the field . " fields " : { " abstract " : { " classattribute_id " : 2643 " description " : " " " id " : 7657698 " identifier " : " abstract " " name " : " Abstract ( descrizione breve ) " " string_value " : " Personale in servizio al 31 / 12 / 13 .