def parse_profile_html(html):
    '''Parses a user profile page into the data fields and essays.

    It turns out parsing HTML, with all its escaped characters, and handling
    wacky unicode characters, and writing them all out to a CSV file (later),
    is a pain in the ass because everything that is not ASCII goes out of its
    way to make your life hard.

    During this function, we handle unescaping of html special
    characters. Later, before writing to csv, we force everything to ASCII,
    ignoring characters that don't play well.
    '''
    html = html.lower()
    soup = BeautifulSoup(html, convertEntities=BeautifulSoup.HTML_ENTITIES)
    NA = 'NA'
    basics_soup = soup.find(name = 'div', attrs = {'id': 'basic_info'})
    details_soup = soup.find(name='div', attrs={'id':'profile_details'})
    essays_soup = soup.find(name='div', attrs={'id':'main_column'})
    if not (basics_soup and details_soup and essays_soup):
        print 'Profile likely deleted. Missing expected html structure.'
        return None, None
    profile = {}
    # Extract top-line profile data (age, gender, city) from tags like
    # <span id='ajax_gender'>Female</span>
    for tag in basics_soup.findAll(name = 'span')[1:]:  # Skip 'Last Online'
        feature, value = tag['id'].split('_', 1)[1], tag.text.strip()
        print feature, value
        profile[feature] = value
    # Extract personal data items from tags like
    # <dd id='ajax_bodytype'>Female</span>
    for tag in details_soup.findAll(name = 'dd'):
        try:
            feature, value = tag['id'].split('_', 1)[1], tag.text.strip()
            if feature == 'height':
                # Special case height to parse into inches
                print value
                feet, inches = [int(x[:-1]) for x in value.split()[:2]]
                value = str(int(feet) * 12 + int(inches))
            print feature, value
            profile[feature] = value
        except __HOLE__:
            continue
    # Essays
    essays = {}
    for e in essays_soup.findAll('div', recursive=False):
        if e['id'].startswith('essay_'):
            essay_id = int(e['id'].split('essay_')[1])
            title = e.a.text
            user_response = ''.join(
                str(x) for x in e.div.div.contents).replace('<br />', '').strip()
            essays[essay_id] = (title, user_response)
        elif e['id'] == 'what_i_want':
            # These are high-level details about what the user wants in a date
            # TODO: parse and incorporate these as profile features
            pass
    return profile, essays
dataset/ETHPy150Open everett-wetchler/okcupid/FetchProfiles.py/parse_profile_html