# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: hackdartmouth-v
#     language: python
#     name: hackdartmouth-v
# ---

import os
import pandas as pd
import numpy as np
import nltk


# ### Using corpus of names

def format_names():
    female_fn = '../names/female.txt'
    male_fn = '../names/male.txt'

    males = pd.read_csv(male_fn, skiprows=4)
    males.rename(columns={males.columns[0]:'MaleNames'}, inplace=True)

    females = pd.read_csv(female_fn, skiprows=4)
    females.rename(columns={females.columns[0]:'FemaleNames'}, inplace=True)

    merged = females.combine_first(males)
    merged = merged.applymap(lambda s:s.lower() if type(s) == str else s)
    
    return merged


merged = format_names()


def check_name_sex(name, merged):
    name = name.lower()
    maleflags = merged['MaleNames'].eq(name).any()
    femaleflags = merged['FemaleNames'].eq(name).any()
    if maleflags and femaleflags:
        val = 'ambiguous'
    elif maleflags:
        val = 'male'
    elif femaleflags:
        val = 'female'
    else:
        val = 'unknown'
    print('The name {0} was decided to be {1}'.format(name, val))
    return val
        



benchmark_names = ['john', 'mary', 'jean', 'ali', 'susan','mamta','harry','hermione','ron','peter','marvin','julie','jeremy','salvador']

for i in benchmark_names:
    check_name_sex(i, merged)

# ### Using gender guesser

# +
import gender_guesser.detector as gender
d = gender.Detector()

for i in benchmark_names:
    #check_str = unicode(i.capitalize(), "utf-8")
    #check_str = i.capitalize().decode("utf-8")
    print("The name {0} was decided to be {1}".format(i, d.get_gender(i.capitalize())))
# -

# ### Reading from bioarxiv

from bs4 import BeautifulSoup
import requests
import re

page_link = 'https://t.co/EtiFK4DUZe'

# scratch space

# +
import pandas as pd

page_response = requests.get('https://t.co/EtiFK4DUZe')
page_content = BeautifulSoup(page_response.content, "html.parser")
all_meta_info = page_content.find_all("meta")
listify = list(all_meta_info)

#names = [name for name in listify if 'Contributor' in name]
#soup = BeautifulSoup(page_response,'lxml')
#table = soup.find_all('table')[0] 
#df = pd.read_html(str(table))
#df.head()
# -



# +
def words_in_line(l):
    words = re.findall(r'"(.*?)"', str(l))
    return words
    
vals = list(map(words_in_line, listify))
# -

# "DC.Date"
date_info = [v[0] for v in vals if "DC.Date" in v]
title_info = [u[0] for u in vals if "citation_title" in u]

page_content

allnames = [n[0] for n in vals if 'DC.Contributor' in n]

allnames


# end of scratch space

def extract_name(page_link):
    page_response = requests.get(page_link, timeout=5)
    page_content = BeautifulSoup(page_response.content, "html.parser")
    all_meta_info = page_content.find_all("meta")
    meta_name = all_meta_info[14]
    actual_name = re.findall(r'"(.*?)"', str(meta_name))[0]
    first_name = actual_name.split(" ")[0]
    return first_name


name_link = extract_name(page_link)


def retweet_or_no(link):
    df = format_names()
    name = extract_name(link)
    label = check_name_sex(name, df)
    if label == 'female':
        return True
    return False


tweet = retweet_or_no(page_link)
print(tweet)

StarcoderdataJupyterScriptsDedupFiltered