Scraping all closed captions (subtitles) of a YouTube's creator's video library A script that scrapes all YouTube's videos closed captions(subtitles) of a single creator's video library. It scrapes either auto-generated or manually written subtitles. The script accepts a link to the creators video library, such as, "https://www.youtube.com/user/creator/videos" and outputs subtitles of each video to it own respective '.txt' file. I use Selenium to trigger the events and BeautifulSoup to scrape the content. 



import time
from bs4 import BeautifulSoup
import urllib.request
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import re
import sys

# Clicks on "Load More" button to display all users videos. 
def display_all_videos(driver):
    while(True):
        try:
            element = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CLASS_NAME, "yt-uix-load-more")))
            element.click()
        except:
            break

# Creates a list of tuples (video_title, video_link) of all 
# videos displayed on page 
def video_list(driver):
    videos = driver.find_elements_by_class_name("yt-uix-tile-link")
    video_links = []
    for vid in videos:
        video_links.append((vid.text,vid.get_attribute("href")))
    return video_links

# Clicks on CC(Closed Caption) button in YouTube video
def enable_subtitles(driver):
    elem = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CLASS_NAME, "ytp-subtitles-button")))
    elem.click()

def subtitles_link(driver):
    time.sleep(1)
    timings = driver.execute_script("return window.performance.getEntries();")

    # Find string in timings that contains the substring 'srv3'
    # which is the subtitles link.
    link = ""
    for t in timings:
         for v in t.values():
             if "srv3" in str(v):
                 link = v
    return link

def create_file(title, link,subtitles):
    # remove illegal chars for file name
    title = "".join([c for c in filename if c.isalpha() or c.isdigit() or c==' ']).rstrip()

    try:
        file = open(title + '.txt', 'w')    
        file.write('LINK: ' + link + '\n')
        file.write(subtitles)
        file.close()
    except:
        print("Can't create file for: " + title + " : " + link)

def scrape_subtitles(subtitle_link):
    r = urllib.request.urlopen(subtitle_link).read()
    soup = BeautifulSoup(r)

    # Remove tags (&lt;*&gt;), \n, and unecessary whitespace 
    s = re.sub(r'&lt;.+?&gt;', '', soup.prettify())   
    s = re.sub(r'\n', '', s)                    
    s = re.sub( '\s+', ' ', s ).strip()         
    return s

def main(argv):
    driver = webdriver.Chrome('C:\Program Files\ChromeDriver\chromedriver.exe')

    # Visit page and load all videos to create a list of
    # tuples(video_name,video_link) of the videos 
    driver.get(argv[1])
    display_all_videos(driver)
    videos = video_list(driver)

    # Visit video's page and enable 'CC' to scrape the subtitles and 
    # save subtitles to '.txt' file. 
    for v in videos:
        driver.get(v[1])
        try:
            enable_subtitles(driver)
            link = subtitles_link(driver)
            subtitles = scrape_subtitles(link)
        except:
            subtitles = "No Closed Caption"
        create_file(v[0],v[1],subtitles)

if __name__ == "__main__":
    main(sys.argv)





Readable?
Follows Python's best practices?
Bug

This was probably a posting or a copy/paste bug - in create_file() you meant to name the first parameter to be filename instead of title.

Code Style and PEP8 notes


use more descriptive variable names - r, s, c, v are not good variable names
remove unused TimeoutException import, organize imports in groups and alphabetically (PEP8 reference)
move the comments before the functions into proper docstrings
avoid catching broad exceptions via bare except
follow PEP8 whitespace and newline usage guidelines - specifically, whitespaces around the operators, two newlines between the function definitions 


Third-party Library Usage notes


BeautifulSoup constructor accepts a file-like object as well - you don't have to explicitly call .read() method
it is a good idea to specify BeautifulSoup's parser explicitly:

soup = BeautifulSoup(r, "html.parser")  # or "html5lib", or "lxml"


Otherwise, BeautifulSoup would pick a parser automatically depending on what is available. This would mean that on your machine it may pick, say, lxml - on someone else's - html5lib, on other's html.parser. And, there might be different results since different parser behave differently.
a bigger problem is that you are not actually using BeautifulSoup capabilities at all - you are post-processing the "soup" with regular expressions. It looks like you can achieve the same by calling the .get_text() method on the "soup" object
for better reusability I would probably assume chromedriver is in PATH instead of hardcoding the path to it explicitly
you can reuse WebDriverWait instance instead of re-instantiating it multiple times
you never close the driver instance
time.sleep() is generally unreliable when used for waiting for something with selenium - recheck if you actually need it 


Code Organization notes

You are passing driver to multiple methods - it may be a good idea to have a class with a driver and wait attributes for better reusability.

Some Syntax Sugar


you can construct a list of videos with a list comprehension:

[(video.text, video.get_attribute("href"))
 for video in driver.find_elements_by_class_name("yt-uix-tile-link")]

you can unpack the list of videos into filename and a link:

for filename, link in videos:



Modified Code:

import sys
import time
import urllib.request

from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait


class YoutubeSubtitlesScraper:
    def __enter__(self):
        self.driver = webdriver.Chrome()

        self.wait = WebDriverWait(self.driver, 10)

        self.driver.get(start_url)
        self.display_all_videos()

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.driver.close()

    def display_all_videos(self):
        """Clicks on "Load More" button to display all users videos."""
        while True:
            try:
                element = self.wait.until(EC.element_to_be_clickable((By.CLASS_NAME, "yt-uix-load-more")))
                element.click()
            except TimeoutException:
                break

    def subtitles(self):
        """Visits video's page, enables 'CC' to scrape the subtitles and generates filename, link and the subtitles content."""
        videos = [(video.text, video.get_attribute("href"))
                  for video in self.driver.find_elements_by_class_name("yt-uix-tile-link")]

        for filename, link in videos:
            self.driver.get(link)
            self.enable_subtitles()

            link = self.get_subtitles_link()
            yield filename, link, self.scrape_subtitles(link) if link else "No Closed Caption"

    def enable_subtitles(self):
        """Clicks on CC(Closed Caption) button in YouTube video."""
        show_subtitles_button = self.wait.until(EC.element_to_be_clickable((By.CLASS_NAME, "ytp-subtitles-button")))
        show_subtitles_button.click()

    def get_subtitles_link(self):
        """Finds string in performance timings that contains the substring 'srv3' which is the subtitles link."""
        time.sleep(1)
        timings = self.driver.execute_script("return window.performance.getEntries();")

        for timing in timings:
            for value in timing.values():
                if "srv3" in str(value):
                    return value
        return ""

    def scrape_subtitles(self, subtitle_link):
        """HTML parses subtitles."""
        response = urllib.request.urlopen(subtitle_link)
        soup = BeautifulSoup(response, "lxml")
        return soup.get_text(strip=True)


def create_file(filename, link, subtitles):
    """Creates file for the subtitle."""
    title = "".join([c for c in filename if c.isalpha() or c.isdigit() or c == ' ']).rstrip()

    with open(title + '.txt', 'w') as subtitles_file:
        subtitles_file.write('LINK: ' + link + '\n')
        subtitles_file.write(subtitles)


if __name__ == "__main__":
    start_url = sys.argv[1]

    with YoutubeSubtitlesScraper(start_url) as scraper:
        for filename, link, content in scraper.subtitles():
            try:
                create_file(filename, link, content)
            except:
                print("Can't create file for: " + filename + " : " + link)


(not tested)

Note how we made the class a context manager by defining the __enter__() and __exit__() magic methods.