Scrape every Java themed codingbat problem set and your personal solutions I made the switch from JavaScript to Python lately. This is my first Python project and my first web-scraper. Do I have good practices? And could this be optimized?
from bs4 import BeautifulSoup
import requests


urlHalf = 'codingbat.com'

print(&quot;what is your email&quot;)
inputOne = input(&quot;&gt;&quot;)
print(&quot;password&quot;)
inputTwo = input(&quot;&gt;&quot;)

post_params = {
    &quot;uname&quot;: inputOne,
    &quot;pw&quot;: inputTwo,
}

# urls to use
base_url = &quot;https://codingbat.com&quot;
login_url = &quot;https://codingbat.com/login&quot;
java_url = &quot;https://codingbat.com/java/&quot;


def get_href():
    print(&quot;Getting directories&quot;)
    href_array = []
    initial = requests.get(base_url)
    soup = BeautifulSoup(initial.content, &quot;lxml&quot;)
    tab_data = soup.find(class_=&quot;floatleft&quot;)
    hrefs = tab_data.find_all(class_=&quot;h2&quot;)
    for href in hrefs:
        href_array.append(href.text)
    return(href_array)


def gather_links():
    directories = get_href()
    print(&quot;Getting problem links&quot;)
    link_array = []
    for directory in directories:
        r = requests.get(java_url + directory)
        # get links to problems
        content = BeautifulSoup(r.content, &quot;lxml&quot;)
        links = content.find_all(
            &quot;a&quot;, href=lambda href: href and &quot;prob&quot; in href)
        link_array.append(links)
    return(link_array)


def join_links():
    linkList = gather_links()
    new_link_list = {}
    print(&quot;Joining links&quot;)
    for link in linkList:
        for each in link:
            text = each.text
            backUrl = each.get(&quot;href&quot;)
            realUrl = &quot;http://&quot; + urlHalf + backUrl
            new_link_list.update({text: realUrl})
    return new_link_list


def scrape_data():
    f = open('answers/answers.java', 'w')
    linkList = join_links()
    with requests.Session() as session:
        post = session.post(login_url, data=post_params)
    for i in linkList:
        print(&quot;Scraping: &quot; + i)
        response = session.get(linkList[i])
        content = BeautifulSoup(response.content, &quot;lxml&quot;)
        indentDiv = content.find(class_=&quot;indent&quot;)
        table = indentDiv.find(&quot;form&quot;, {&quot;name&quot;: &quot;codeform&quot;})
        aceDiv = table.find(&quot;div&quot;, id=&quot;ace_div&quot;)
        f.write(&quot;//&quot; + i + &quot;: &quot; + linkList[i] + &quot;\n\n&quot;)
        f.write(aceDiv.text + &quot;\n&quot;)
    print(&quot;done&quot;)


if __name__ == &quot;__main__&quot;:
    scrape_data()
Global nomenclature
urlHalf should be URL_HALF. That said, a better name would be DOMAIN, since that's what that string actually is.
Further, after declaring this you don't use it correctly; these:
base_url = &quot;https://codingbat.com&quot;
login_url = &quot;https://codingbat.com/login&quot;
java_url = &quot;https://codingbat.com/java/&quot;

could be
BASE_URL = 'https://' + DOMAIN
LOGIN_URL = BASE_URL = '/login'
JAVA_URL = BASE_URL + '/java'

Input
print(&quot;what is your email&quot;)
inputOne = input(&quot;&gt;&quot;)
print(&quot;password&quot;)
inputTwo = input(&quot;&gt;&quot;)

should not occur at global scope; it should occur in a function. Also, the use of input for passwords is insecure; use getpass instead.
Return parens
These parens:
return(href_array)

are redundant.
Generator
link_array = []
for directory in directories:
    # ...
    link_array.append(links)
return(link_array)

can simply be
for directory in directories:
    # ...
    yield links

or, if you want a &quot;flattened&quot; output that does not have nested lists, use yield from links.
File extensions
Does this file:
f = open('answers/answers.java', 'w')

actually receive Java code? It seems not, in which case - use just .txt.