**Task**

I have a large spreadsheet where each line should include:

* The URL of a social media account
* A field indicating whether the account is "active"
* A name and UID number for each account

I have to find if the activity field is accurate for every account on the list, and fix any mismatches.

---

**Code**

```
##### IMPORTS

import requests
from bs4 import BeautifulSoup
import datetime
import re
from selenium import webdriver
import tkinter
import csv
from time import sleep

##### METHODS TO SCRAPE DATE OF MOST RECENT ACTIVITY FROM SOCIAL PAGES

# A method to get the date of last activity from a given Twitter feed.
def getMostRecentActivityTwitter(url):

    # Grab the page, raising an exception if it doesn't work
    response = requests.get(url)
    if '40' in str(response):
        raise Exception('Couldn''t find page!')

    # Parse the response into Beautiful Soup
    html = response.text
    soup = BeautifulSoup(html, 'html.parser')

    # Grab all the tweets
    tweets = soup.find_all('div', class_='tweet')

    # Take a look at the first two (if there's a pinned tweet, we want both)
    try:
        tweet1 = tweets[0]
        tweet2 = tweets[1]

        # Get their dates
        dateString1 = tweet1.find('a', class_='tweet-timestamp').attrs['title'].split('-')[1]
        dateString2 = tweet2.find('a', class_='tweet-timestamp').attrs['title'].split('-')[1]
        date1 = datetime.datetime.strptime(dateString1, ' %d %b %Y')
        date2 = datetime.datetime.strptime(dateString2, ' %d %b %Y')

        # Return the more recent of the two
        if date1-date2>datetime.timedelta(0):
            return date1
        else:
            return date2

    # Raise an exception if it didn't work
    except Exception:
        raise Exception('Parsing failed!')

# A method to get the date of last activity from a given Facebook page.
def getMostRecentActivityFB(url):

    # Grab the page, raising an exception if it doesn't work
    response = requests.get(url)
    if '40' in str(response):
        raise Exception('Couldn''t find page!')

    # Parse the response into Beautiful Soup
    html = response.text
    soup = BeautifulSoup(html, 'html.parser')

    # Grab the most recent post's timestamp
    try:
        mostRecentTimestamp = soup.find('abbr', class_=re.compile('_5ptz'))

        # Get an appropriately formatted datetime object out of it
        dateList = mostRecentTimestamp.attrs['title'].split(' ')
        dateString = dateList[1]+' '+dateList[2]+' '+dateList[3]
        date = datetime.datetime.strptime(dateString, '%B %d, %Y')

        # Return the result
        return date

    # Raise an exception if it didn't work
    except Exception:
        raise Exception('Parsing failed!')

# A method to get the date of last activity from a given YouTube channel.
def getMostRecentActivityYouTube(url):

    # Reformat the URL to make sure we're getting the Videos page
    splitURL = url.split('/')
    baseURL = splitURL[2]
    pageType = splitURL[3]
    channelName = splitURL[4]
    fixedURL = 'https://'+baseURL+'/'+pageType+'/'+channelName+'/videos'

    # Grab the page, raising an exception if it doesn't work
    response = requests.get(fixedURL)
    if '40' in str(response):
        raise Exception('Couldn''t find page!')

    # Parse the response into Beautiful Soup
    html = response.text
    soup = BeautifulSoup(html, 'html.parser')

    # Get the 'x units ago' timestamp from the most recent video
    try:
        howRecentString = soup.find(text=re.compile('ago'))

        # Turn it into a timedelta object
        howRecentList = howRecentString.split(' ')
        amount = howRecentList[0]
        unit = howRecentList[1]
        if unit[-1] != 's':
            unit = unit + 's'
        if unit == 'weeks':
            unit = 'days'
            amount = str(int(amount)*7)
        elif unit == 'months':
            unit = 'days'
            amount = str(int(amount)*30)
        elif unit == 'years':
            unit = 'days'
            amount = str(int(amount)*365)
        howRecent = datetime.timedelta(**{unit:float(amount)})

        # Calculate the date from today's, and return it
        todaysDate = datetime.datetime.today()
        date = (todaysDate - howRecent).replace(hour=0,minute=0,second=0,microsecond=0)
        return date

    # Raise an exception if it didn't work
    except Exception:
        raise Exception('Parsing failed!')

# A method to get the date of last activity from a given Instagram page.
def getMostRecentActivityInstagram(url):

    # Use Selenium to open the page, and read it into Beautiful Soup
    driver = webdriver.Ie()
    driver.get(url)
    soup = BeautifulSoup(driver.page_source, 'html.parser')
    driver.quit()

    # Find the link to the most recent post
    try:
        href = soup.find('a', href=re.compile('/p')).attrs['href']

        # Build the URL for the most recent post
        splitHREF = href.split('/')
        newURL = 'https://instagram.com/p/'+splitHREF[2]

        # Use Selenium to open the post, and read it into Beautiful Soup
        driver = webdriver.Ie()
        driver.get(newURL)
        newSoup = BeautifulSoup(driver.page_source, 'html.parser')
        driver.quit()

        # Find, interpret, and return the timestamp associated with the post
        dateString = newSoup.find('time').attrs['title']
        date = datetime.datetime.strptime(dateString, '%b %d, %Y')
        return date

    # Raise an exception if it didn't work
    except Exception:
        raise Exception('Parsing failed!')

# A method to get the date of last activity from a given Google Plus page.
def getMostRecentActivityGP(url):

    # Grab the page, raising an exception if it doesn't work
    response = requests.get(url)
    if '40' in str(response):
        raise Exception('Couldn''t find page!')

    # Parse the response into Beautiful Soup
    html = response.text
    soup = BeautifulSoup(html, 'html.parser')

    # Find, interpret, and return the timestamp of the most recent post
    try:
        dateString = soup.find('a', class_='o-U-s').text
        date = datetime.datetime.strptime(dateString, '%Y-%m-%d')
        return date

    # Raise an exception if it didn't work
    except Exception:
        raise Exception('Parsing failed!')

# A method to get the date of last activity from a given LinkedIn page.
def getMostRecentActivityLinkedIn(url):

    # Use Selenium to open the page, and read it into Beautiful Soup
    driver = webdriver.Ie()
    driver.get(url)
    soup = BeautifulSoup(driver.page_source, 'html.parser')
    driver.quit()

    # Get the 'x units ago' timestamp from the most recent post
    try:
        howRecentString = soup.find('a', class_='nus-timestamp').text

        # Turn it into a timedelta object
        howRecentList = howRecentString.split(' ')
        amount = howRecentList[0]
        unit = howRecentList[1]
        if unit[-1] != 's':
            unit = unit + 's'
        if unit == 'weeks':
            unit = 'days'
            amount = str(int(amount)*7)
        elif unit == 'months':
            unit = 'days'
            amount = str(int(amount)*30)
        elif unit == 'years':
            unit = 'days'
            amount = str(int(amount)*365)
        howRecent = datetime.timedelta(**{unit:float(amount)})

        # Calculate the date from today's, and return it
        todaysDate = datetime.datetime.today()
        date = (todaysDate - howRecent).replace(hour=0,minute=0,second=0,microsecond=0)
        return date

    # Raise an exception if it didn't work
    except Exception:
        raise Exception('Parsing failed!')

# A method to get the date of last activity from a given Tumblr page.
def getMostRecentActivityTumblr(url):

    # Grab the page, raising an exception if it doesn't work
    response = requests.get(url)
    if '40' in str(response):
        raise Exception('Couldn''t find page!')

    # Parse the response into Beautiful Soup
    html = response.text
    soup = BeautifulSoup(html, 'html.parser')

    # Build the URL for the most recent post
    try:
        splitURL = url.split('/')
        baseURL = splitURL[2]
        postID = soup.find('article').attrs['data-post-id']
        newURL = 'http://'+baseURL+'/post/'+postID

        # Try to grab it, and parse the response into Beautiful Soup
        newResponse = requests.get(newURL)
        if '40' in str(newResponse):
            raise Exception('Couldn''t find page!')
        newHTML = newResponse.text
        newSoup = BeautifulSoup(newHTML, 'html.parser')

        # Find, interpret, and return the date of the post
        dateString = newSoup.find('a', class_=re.compile('post-date')).text
        dateList = dateString.split(',')
        fixedDateString = dateList[0][0:-2]+dateList[1]
        date = datetime.datetime.strptime(fixedDateString, '%b %d %Y')
        return date

    # Raise an exception if it didn't work
    except Exception:
        raise Exception('Parsing failed!')

# A method to get the date of last activity from a given Vkontakte page.
def getMostRecentActivityVK(url):

    # Grab the page, raising an exception if it doesn't work
    response = requests.get(url)
    if '40' in str(response):
        raise Exception('Couldn''t find page!')

    # Parse the response into Beautiful Soup
    html = response.text
    soup = BeautifulSoup(html, 'html.parser')

    # Find the date of the most recent post
    try:
        dateString = soup.find('a', class_='wi_date').text

        # Figure out the date from the string
        if 'ago' in dateString:
            date = datetime.datetime.today()
        elif 'today' in dateString:
            date = datetime.datetime.today()
        elif 'yesterday' in dateString:
            date = datetime.datetime.today() - datetime.timedelta(days=1)
        elif ':' in dateString:
            dateList= dateString.split(' ')
            fixedDateString = str(dateList[0])+' '+str(dateList[1])+' '+str(datetime.datetime.today().year)
            date = datetime.datetime.strptime(fixedDateString, '%d %b %Y')
        else:
            date = datetime.datetime.strptime(dateString, '%d %b %Y')

        # Clean up the date and return it
        date = date.replace(hour=0,minute=0,second=0,microsecond=0)
        return date

    # Raise an exception if it didn't work
    except Exception:
        raise Exception('Parsing failed!')

##### OTHER METHODS

# A method to clean up URL's
def cleanURL(url):
    strippedURL = url.strip().lower()
    coreURL = strippedURL.split('//')[-1].split('?')[0].strip('www.').strip('/')
    rebuiltURL = 'https://'+coreURL
    if 'facebook' in coreURL:
        rebuiltURL = rebuiltURL+'/?brand_redir=DISABLE'
    return(rebuiltURL)

# A method to check a given URL, and write the results to the output table
def checkAndRecord(rowID, accountName, url, putativeActivity, writer):

    # Grab the domain name part of the URL
    domainName = url.split('/')[2]

    # If it's from one of the services we can work with...
    if 'twitter' in domainName:

        # Attempt to scrape the date...
        try:
            date = getMostRecentActivityTwitter(url)

        # ...or record the error if we can't
        except Exception as errorMessage:
            writer.writerow([rowID, accountName, url, 'Not Checked', errorMessage, ''])
            return

    # Do the same for each service
    elif 'facebook' in domainName:
        try:
            date = getMostRecentActivityFB(url)
        except Exception as errorMessage:
            writer.writerow([rowID, accountName, url, 'Not Checked', errorMessage, ''])
            return
    elif 'youtube' in domainName:
        try:
            date = getMostRecentActivityYouTube(url)
        except Exception as errorMessage:
            writer.writerow([rowID, accountName, url, 'Not Checked', errorMessage, ''])
            return
    elif 'instagram' in domainName:
        try:
            date = getMostRecentActivityInstagram(url)
        except Exception as errorMessage:
            writer.writerow([rowID, accountName, url, 'Not Checked', errorMessage, ''])
            return
    elif 'google' in domainName:
        try:
            date = getMostRecentActivityGP(url)
        except Exception as errorMessage:
            writer.writerow([rowID, accountName, url, 'Not Checked', errorMessage, ''])
            return
    elif 'linkedin' in domainName:
        try:
            date = getMostRecentActivityLinkedIn(url)
        except Exception as errorMessage:
            writer.writerow([rowID, accountName, url, 'Not Checked', errorMessage, ''])
            return
    elif 'tumblr' in domainName:
        try:
            date = getMostRecentActivityTumblr(url)
        except Exception as errorMessage:
            writer.writerow([rowID, accountName, url, 'Not Checked', errorMessage, ''])
            return
    elif 'vk' in domainName:
        try:
            date = getMostRecentActivityVK(url)
        except Exception as errorMessage:
            writer.writerow([rowID, accountName, url, 'Not Checked', errorMessage, ''])
            return

    # If it's not from one of our services, record that in the output
    else:
        writer.writerow([rowID, accountName, url, 'Not Checked', 'No scraper for this service!', ''])
        return

    # If we managed to get a date, check if it's active...
    if datetime.datetime.today() - date < datetime.timedelta(days=365):

        # ...check if it's supposed to be dormant, and record the results...
        if putativeActivity == 'Dormant':
            writer.writerow([rowID, accountName, url, 'Active', '', 'Yes'])
            return
        else:
            writer.writerow([rowID, accountName, url, 'Active', '', 'No'])
            return

    # ...or do the same if it's dormant
    else:
        if putativeActivity == 'Active':
            writer.writerow([rowID, accountName, url, 'Inactive', '', 'Yes'])
            return
        else:
            writer.writerow([rowID, accountName, url, 'Inactive', '', 'No'])
            return

##### MAIN METHOD

# Make and hide tkinter window
root = tkinter.Tk()
root.withdraw()

# Prompt user for input file, and open it
csvToRead = tkinter.filedialog.askopenfile(parent=root, title='Select input csv file.')
with open(csvToRead.name, 'r', encoding='utf8') as fileToRead:
    reader = csv.reader(fileToRead)
    next(reader) # Skip the first line (header)

    # Prompt user for output directory, and initialize output file
    outputDir = tkinter.filedialog.askdirectory(parent=root, title='Select output directory:')
    with open(outputDir+'/recency_check_output.csv', 'w', encoding='utf8') as fileToWrite:
        writer = csv.writer(fileToWrite, lineterminator='\n')
        writer.writerow(['ID', 'Account Name', 'URL', 'Activity', 'Reason', 'Mismatch?'])

        # For each row of the table...
        for row in reader:

            # Get and clean the URL
            url = cleanURL(row[3])

            # Assign the other stuff we care about
            rowID = row[0]
            accountName = row[2]
            putativeActivity = row[8]

            # Keep track of what number we're on
            print(rowID)

            # Wait a second to avoid tripping bot detectors
            sleep(1)

            # Check the page's activity and write the results to the output table
            checkAndRecord(rowID, accountName, url, putativeActivity, writer)

```

---

**Concerns**

I'm looking for general pointers and advice on how to improve my code... both this specific code, and my coding style in general. Some specific concerns I have:

* Because each social media service presents its post time-stamps differently, I've had to write a separate method for each one. However, some logic is duplicated, are there ways I could consolidate this to make my code more concise?
* Are there better scraping methods I could use?
* Is there a better way to manipulate the text?
* Am I using exceptions properly?
* Do I comment too much? Or make bad comments?