load_and_preprocess.py
def parse_and_load_course_branch_item(course_data_path, conn, course_zip_name):
    """take all of the course branch item content and create vocabulary
    """
    content_path = os.path.join(course_data_path, "course_branch_item_content")
    course_slug = course_zip_name.replace("_", "-")

    sql_select_course_id = (
        "SELECT DISTINCT course_branch_items.course_branch_id, " +
        "course_item_id, course_branch_module_name, " +
        "course_branch_lesson_name, course_branch_item_name FROM " +
        "course_branch_modules, course_branch_lessons, course_branch_items, " +
        "course_branches, courses WHERE course_slug = (?) " +
        "AND courses.course_id == course_branches.course_id " +
        "AND course_branches.course_branch_id == course_branch_items.course_branch_id " +
        "AND course_branch_items.course_lesson_id == course_branch_lessons.course_lesson_id " +
        "AND course_branch_lessons.course_module_id == course_branch_modules.course_module_id"
    )

    c = conn.cursor()
    c.execute(sql_select_course_id, (course_slug,))

    # module name > lesson name > item name > to processed vocabulary (list of words)
    course_vocabulary = {}

    rows = c.fetchmany()
    while rows:
        for row in rows:
            (course_branch_id, course_item_id, course_branch_module_name,
             course_branch_lesson_name, course_branch_item_name,) = row
            # load the raw json file for branch item
            course_branch_item_path = os.path.join(
                content_path, "{}-{}.json".format(course_branch_id, course_item_id))
            with open(course_branch_item_path, "r") as cbif:
                # attempt to load the json file, otherwise continue
                try:
                    raw_cbi = json.load(cbif)
                except Exception as e:
                    print(e)
                    continue

                try:
                    if raw_cbi["message"] == "" and raw_cbi["statusCode"] == 204 and raw_cbi["reason"] == "ignore assesments":
                        continue
                except KeyError:
                    pass

                try:
                    if raw_cbi["message"] == "" and raw_cbi["statusCode"] == 404:
                        continue
                except KeyError:
                    pass

                try:
                    if raw_cbi["message"] == None and raw_cbi["errorCode"] == "Not Authorized":
                        continue
                except KeyError:
                    pass

                try:
                    if raw_cbi["message"].startswith("No item ItemId(") and raw_cbi["errorCode"] == None:
                        continue
                except KeyError:
                    pass

                normalized_processed_text = None

                try:
                    # try to get the definition value of the item
                    definition_raw_html = raw_cbi["linked"]["openCourseAssets.v1"][0]["definition"]["value"]
                    definition_text = " ".join(BeautifulSoup(
                        definition_raw_html, "html.parser").stripped_strings)
                    normalized_processed_text = preprocess_string(
                        definition_text)
                    update_course_vocabulary(
                        course_vocabulary, course_branch_module_name,
                        course_branch_lesson_name, course_branch_item_name,
                        normalized_processed_text)
                    continue
                except KeyError:
                    pass

                try:
                    # check if the branch item is a video with subtitles, get subtitles
                    subtitles_lookup = raw_cbi["linked"]["onDemandVideos.v1"][0]["subtitlesTxt"]
                    if not subtitles_lookup.keys():
                        continue  # no subtitles for the video
                    subtitle_filepath = course_branch_item_path + ".subtitles.txt"
                    with open(subtitle_filepath, "r") as subfp:
                        subtitle_raw_text = "".join(subfp.readlines())
                        normalized_processed_text = preprocess_string(
                            subtitle_raw_text)
                    update_course_vocabulary(
                        course_vocabulary, course_branch_module_name,
                        course_branch_lesson_name, course_branch_item_name,
                        normalized_processed_text)
                    continue
                except KeyError:
                    pass

                raise Error("unhandled cbi")

        rows = c.fetchmany()

    # save the course_vocabulary to disk
    vocab_filepath = os.path.join(
        course_data_path, "..", "vocabulary.{}.json".format(course_slug))
    with open(vocab_filepath, "w") as vocab_file:
        json.dump(course_vocabulary, vocab_file)
take all of the course branch item content and create vocabulary
    
take all of the course branch item content and create vocabulary
take all of the course branch item content and create vocabulary
def parse_and_load_course_branch_item(course_data_path, conn, course_zip_name):
    content_path = os.path.join(course_data_path, "course_branch_item_content")
    course_slug = course_zip_name.replace("_", "-")
    sql_select_course_id = (
        "SELECT DISTINCT course_branch_items.course_branch_id, " +
        "course_item_id, course_branch_module_name, " +
        "course_branch_lesson_name, course_branch_item_name FROM " +
        "course_branch_modules, course_branch_lessons, course_branch_items, " +
        "course_branches, courses WHERE course_slug = (?) " +
        "AND courses.course_id == course_branches.course_id " +
        "AND course_branches.course_branch_id == course_branch_items.course_branch_id " +
        "AND course_branch_items.course_lesson_id == course_branch_lessons.course_lesson_id " +
        "AND course_branch_lessons.course_module_id == course_branch_modules.course_module_id"
    )
    c = conn.cursor()
    c.execute(sql_select_course_id, (course_slug,))
    course_vocabulary = {}
    rows = c.fetchmany()
    while rows:
        for row in rows:
            (course_branch_id, course_item_id, course_branch_module_name,
             course_branch_lesson_name, course_branch_item_name,) = row
            course_branch_item_path = os.path.join(
                content_path, "{}-{}.json".format(course_branch_id, course_item_id))
            with open(course_branch_item_path, "r") as cbif:
                try:
                    raw_cbi = json.load(cbif)
                except Exception as e:
                    print(e)
                    continue
                try:
                    if raw_cbi["message"] == "" and raw_cbi["statusCode"] == 204 and raw_cbi["reason"] == "ignore assesments":
                        continue
                except KeyError:
                    pass
                try:
                    if raw_cbi["message"] == "" and raw_cbi["statusCode"] == 404:
                        continue
                except KeyError:
                    pass
                try:
                    if raw_cbi["message"] == None and raw_cbi["errorCode"] == "Not Authorized":
                        continue
                except KeyError:
                    pass
                try:
                    if raw_cbi["message"].startswith("No item ItemId(") and raw_cbi["errorCode"] == None:
                        continue
                except KeyError:
                    pass
                normalized_processed_text = None
                try:
                    definition_raw_html = raw_cbi["linked"]["openCourseAssets.v1"][0]["definition"]["value"]
                    definition_text = " ".join(BeautifulSoup(
                        definition_raw_html, "html.parser").stripped_strings)
                    normalized_processed_text = preprocess_string(
                        definition_text)
                    update_course_vocabulary(
                        course_vocabulary, course_branch_module_name,
                        course_branch_lesson_name, course_branch_item_name,
                        normalized_processed_text)
                    continue
                except KeyError:
                    pass
                try:
                    subtitles_lookup = raw_cbi["linked"]["onDemandVideos.v1"][0]["subtitlesTxt"]
                    if not subtitles_lookup.keys():
                        continue  
                    subtitle_filepath = course_branch_item_path + ".subtitles.txt"
                    with open(subtitle_filepath, "r") as subfp:
                        subtitle_raw_text = "".join(subfp.readlines())
                        normalized_processed_text = preprocess_string(
                            subtitle_raw_text)
                    update_course_vocabulary(
                        course_vocabulary, course_branch_module_name,
                        course_branch_lesson_name, course_branch_item_name,
                        normalized_processed_text)
                    continue
                except KeyError:
                    pass
                raise Error("unhandled cbi")
        rows = c.fetchmany()
    vocab_filepath = os.path.join(
        course_data_path, "..", "vocabulary.{}.json".format(course_slug))
    with open(vocab_filepath, "w") as vocab_file:
        json.dump(course_vocabulary, vocab_file)
def parse_and_load_course_branch_item ( course_data_path , conn , course_zip_name ) : content_path = os . path . join ( course_data_path , "course_branch_item_content" ) course_slug = course_zip_name . replace ( "_" , "-" ) sql_select_course_id = ( "SELECT DISTINCT course_branch_items.course_branch_id, " + "course_item_id, course_branch_module_name, " + "course_branch_lesson_name, course_branch_item_name FROM " + "course_branch_modules, course_branch_lessons, course_branch_items, " + "course_branches, courses WHERE course_slug = (?) " + "AND courses.course_id == course_branches.course_id " + "AND course_branches.course_branch_id == course_branch_items.course_branch_id " + "AND course_branch_items.course_lesson_id == course_branch_lessons.course_lesson_id " + "AND course_branch_lessons.course_module_id == course_branch_modules.course_module_id" ) c = conn . cursor ( ) c . execute ( sql_select_course_id , ( course_slug , ) ) course_vocabulary = { } rows = c . fetchmany ( ) while rows : for row in rows : ( course_branch_id , course_item_id , course_branch_module_name , course_branch_lesson_name , course_branch_item_name , ) = row course_branch_item_path = os . path . join ( content_path , "{}-{}.json" . format ( course_branch_id , course_item_id ) ) with open ( course_branch_item_path , "r" ) as cbif : try : raw_cbi = json . load ( cbif ) except Exception as e : print ( e ) continue try : if raw_cbi [ "message" ] == "" and raw_cbi [ "statusCode" ] == 204 and raw_cbi [ "reason" ] == "ignore assesments" : continue except KeyError : pass try : if raw_cbi [ "message" ] == "" and raw_cbi [ "statusCode" ] == 404 : continue except KeyError : pass try : if raw_cbi [ "message" ] == None and raw_cbi [ "errorCode" ] == "Not Authorized" : continue except KeyError : pass try : if raw_cbi [ "message" ] . startswith ( "No item ItemId(" ) and raw_cbi [ "errorCode" ] == None : continue except KeyError : pass normalized_processed_text = None try : definition_raw_html = raw_cbi [ "linked" ] [ "openCourseAssets.v1" ] [ 0 ] [ "definition" ] [ "value" ] definition_text = " " . join ( BeautifulSoup ( definition_raw_html , "html.parser" ) . stripped_strings ) normalized_processed_text = preprocess_string ( definition_text ) update_course_vocabulary ( course_vocabulary , course_branch_module_name , course_branch_lesson_name , course_branch_item_name , normalized_processed_text ) continue except KeyError : pass try : subtitles_lookup = raw_cbi [ "linked" ] [ "onDemandVideos.v1" ] [ 0 ] [ "subtitlesTxt" ] if not subtitles_lookup . keys ( ) : continue subtitle_filepath = course_branch_item_path + ".subtitles.txt" with open ( subtitle_filepath , "r" ) as subfp : subtitle_raw_text = "" . join ( subfp . readlines ( ) ) normalized_processed_text = preprocess_string ( subtitle_raw_text ) update_course_vocabulary ( course_vocabulary , course_branch_module_name , course_branch_lesson_name , course_branch_item_name , normalized_processed_text ) continue except KeyError : pass raise Error ( "unhandled cbi" ) rows = c . fetchmany ( ) vocab_filepath = os . path . join ( course_data_path , ".." , "vocabulary.{}.json" . format ( course_slug ) ) with open ( vocab_filepath , "w" ) as vocab_file : json . dump ( course_vocabulary , vocab_file )
take all of the course branch item content and create vocabulary
take all of the course branch item content and create vocabulary
"""take all of the course branch item content and create vocabulary
    """ # module name > lesson name > item name > to processed vocabulary (list of words) # load the raw json file for branch item # attempt to load the json file, otherwise continue # try to get the definition value of the item # check if the branch item is a video with subtitles, get subtitles # no subtitles for the video # save the course_vocabulary to disk
course_data_path
conn
course_zip_name