code: # -*- coding: utf-8 -*-
  """
  Prints out a simple summary of the courses extracted from the school catalogs.
  
  Created on Mon Jun  8 11:51:22 2020
  
  @author: Francisco Vazquez
  """
  
  import os
  import pandas as pd
  
  n_colleges=0
  n_courses = 0
  for filename in os.listdir('../courses'):
  
      # if current file is the one for all schools, skip it
      if filename == "AllSchools.csv":
          continue
  
      csv = pd.read_csv('../courses/'+filename,encoding = 'utf-8')
      n_colleges +=1
      n_courses += len(csv['CourseID'])
      print('number of courses in ' + filename + ': ' + str(len(csv['CourseID'])))
  
  print('\n')
  print('number of colleges: ' + str(n_colleges))
  print("number of courses: " +str(n_courses))
  print("average number of courses per college: " + str((n_courses/n_colleges))) 
  # -*- coding: utf-8 -*-
  
  #Import Statements
  
  import pandas as pd
  import numpy as np
  import nltk
  import spacy
  import courseToBOK
  nltk.download('punkt')
  nltk.download('stopwords')
  nltk.download('wordnet')
  nltk.download('averaged_perceptron_tagger')
  import re
  from re import *
  from nltk.util import ngrams
  from nltk.corpus import stopwords
  import gower
  import matplotlib.pyplot as plt
  from scipy.cluster.hierarchy import linkage, fcluster, dendrogram
  from sklearn.cluster import KMeans
  from sklearn import metrics
  from scipy.spatial.distance import cdist
  from get_df_name_func import get_df_name
  from process_words_funct import process_words
  from tfidf_analysis import tfidf
  from tokenizer import tokenize
  
  
  nlp = spacy.load('en_core_web_sm')
  
  ##Import CSV as Dataframes
  schools_df = pd.read_csv("Smith-07-10-2020-FROZEN.csv")
  del(schools_df['Unnamed: 0'])
  
  
  text_file = open("bok.txt", "r")
  bok = text_file.read().split('\n')
  bok = courseToBOK.cleanBOK(bok)
  for i in range(len(bok)): #lowercase each bok term
    bok[i] = bok[i].lower()
  
  
  
  #retain only courses pertinent to Data Science
  temp_df = []
  for i in range(len(schools_df)): #for each course
    des = str(schools_df['CourseID'][i])
    des = des.lower()
    temp_list = []
    terms = []
    #print(i)
    for w in bok: #for each bok term
      space = r'\s' #regex space pattern
      if re.search(space+w+space,des): #if the course contains the bok term
        if w not in terms: #if the bok term is not already in the list for that description
          terms.append(w) #append bok term
    if len(terms) != 0: #if at least 1 term is in the description
      temp_list = [schools_df['School'][i], schools_df['CourseID'][i], schools_df['Descriptions'][i],', '.join(terms)] #append course to new list
      temp_df.append(temp_list) #append list to new dataframe
  ds_schools_df = pd.DataFrame(temp_df) #create permanent new data frame
  ds_schools_df.columns = ['School','CourseID','Descriptions','Data Science Term'] #label columns
  print("Creating 'csvs/SMITH_THE_courses.csv'...") #create csv of data science courses
  ds_schools_df.to_csv('csvs/SMITH_blank_courses.csv',encoding="utf-8-sig")
  
  #creating new columns with key words
  body = ['Data Science Analytics','Data Science Engineering','Data Management','Research Methods and Project Management','Business Analytics'] #overall headings
  dsana = ['accuracy metrics','data analytics','data analysis','data analytics assessment', 'data lifecycle','data mining','data preparation','graph data analytics',
           'machine learning','natural language processing','open data','operations research', 'optimisation','optimization','predictive analytics','prescriptive analytics',
           'qualitative analytics','reinforced learning','simulation'] #keywords for this category
  dseng = ['big data infrastructures','cloud computing','cloud powered services','collaborative system','continuous improvement cycle','data access',
           'data anonymisation','data driven','data handling','data lifecycle', 'data science engineering','data security','data warehouse solution',
           'devops','dmp','engineering principles','etl','extract transform load','federated access control','ipr protection','nosql','olap','oltp',
           'relational databases','simulation','sql','systems software'] #keywords for this category
  dman = ['data architecture','data archive services','data curation','data factories', 'data formats','data governance strategy','data handling','data integration',
          'data lifecycle','data management','data management plan','data modeling', 'data modeling design','data provenance','data registries','data storage systems',
          'data types','digital libraries','etl','extract transform load','linked data', 'meta data','metadata','meta-data','olap','oltp','open access','open data',
          'open science','operational models','pid'] #keywords for this category
  remeprma = ['data collection','data driven','data lifecycle','data quality evaluation','project management','quality evaluation','research methods','team management',
              'use cases analysis'] #keywords for this category
  busana = ['agile data driven','bpm','business analytics','business intelligence', 'cognitive technologies','crp','customer relations management',
            'data marketing technologies','data driven marketing','data integration analytics', 'data warehouses technologies','econometrics','enterprises','open data',
            'optimization','processes management','use cases analysis','user experience','ux'] #keywords for this category
  bok_cats = {} #dictionary to hold bok categories
  body_keywords = [] #list to hold bok keywords
  #append keyword lists to body_keywords
  body_keywords.append(dsana)
  body_keywords.append(dseng)
  body_keywords.append(dman)
  body_keywords.append(remeprma)
  body_keywords.append(busana)
  i=0
  for i in range(len(body)): #assign each keyword list to their respective category name
    bok_cats[body[i]] = body_keywords[i]
  
  
  diff = ['Website','Data Visualization','Statistics','Experimental Design','Programming','Algorithms/Modeling/AI','Data Collection','Data Sources','Data Types','Data Analysis','Application','Simulation','Software','Lab']
  website = ['information technology','web design','website','network','webpage','web','shiny'] #keywords for this category
  datavis = ['graph','graphs','chart','charts','color theory','barplot','bar plot','visualization', 'visualisation','visualizations','visual','visuals','box plot','boxplot',
             'color','colors','ggplot2','dashboard','boxplot','barplot','pie chart','piechart','tableau'] #keywords for this category
  statistics = ['anova','linear regression','chi squared','probability', 'hypothesis test','regression','statistics','statistic','distribution',
                'variability','variance','percentile','standard deviation','mean','median','mode','average','trendline','trend line'] #keywords for this category
  resmeth = ['research methods','research','research process','design of experiment', 'design of experiments','research question','design study','design studies',
             'research design'] #keywords for this category
  proglang = ['r','python','c++','mysql','c','spss','sql','nosql','programming language', 'programming languages','tools','sas','programming','pandas','package',
              'library','coding','code'] #keywords for this category
  algmodai = ['artificial intelligence','ai','algorithm','algorithms','model','models','modeling', 'machine learning','predict','natural language processing','nlp',
              'topic model','supervised learning','unsupervised learning', 'neural network','k-means cluster','clustering','decision tree'] #keywords for this category
  collect = ['data collection','collect','sample size','raw data','collection','collections'] #keywords for this category
  sources = ['data source','data sources','retrieve','database','databases','import'] #keywords for this category
  types = ['medicalpharmecutical','qualitative','quantitative','econometrics', 'public health','big data','large volumes'] #keywords for this category
  analy = ['data analysis','analyze data','analysis','summary','summarize','summarizing'] #keywords for this category
  applic = ['communication','results','interpretation','application', 'data representation','presentation'] #keywords for this category
  sim = ['simulation','simulations'] #keywords for this category
  software = ['software','github','rstudio','excel','microsoft','gis','colab','jupyter', 'tableau'] #keywords for this category
  lab = ['lab','labs','laboratory'] #keywords for this category
  diff_cats = {} #dictionary to hold diff categories
  diff_keywords = [] #list to hold diff keywords
  #append keyword lists to diff_keywords
  diff_keywords.append(website)
  diff_keywords.append(datavis)
  diff_keywords.append(statistics)
  diff_keywords.append(resmeth)
  diff_keywords.append(proglang)
  diff_keywords.append(algmodai)
  diff_keywords.append(collect)
  diff_keywords.append(sources)
  diff_keywords.append(types)
  diff_keywords.append(analy)
  diff_keywords.append(applic)
  diff_keywords.append(sim)
  diff_keywords.append(software)
  diff_keywords.append(lab)
  i=0
  # this was originally ellie  not sure why
  for i in range(len(diff)): #assign each keyword list to their respective category name
    diff_cats[diff[i]] = diff_keywords[i]
  
  #adding keywords to columns
  for b in body:
    ds_schools_df[b] = 0
  for d in diff:
    ds_schools_df[d] = 0
  
  j=0
  
  for j in range(len(ds_schools_df)): #for each description
    for b in bok_cats: #for each overall category in bok
      i = 0
      for i in range(len(bok_cats[b])): #for each keyword
        comp = str(ds_schools_df['Descriptions'][j]).lower()
        if re.search('\s'+re.escape(bok_cats[b][i])+'\s',comp): #search for keyword in description
          ds_schools_df[b][j] = 1 #assign value of 1 if keyword exists in description
  
    for d in diff_cats: #for each overall category in diff
      i = 0
      for i in range(len(diff_cats[d])): #for each keyword
        comp = str(ds_schools_df['Descriptions'][j]).lower()
        if re.search('\s'+re.escape(diff_cats[d][i])+'\s',comp): #search for keyword in description
          ds_schools_df[d][j] = 1 #assing value of 1 if keyword exists in description
  
  #creating summation column
  ds_schools_df['Sum'] = ds_schools_df.sum(axis=1)
  
  #creating weights
  num = 0
  denom = 1
  frac = float(num/denom)
  
  #create "others" list with all keywords
  others = website
  others.extend(datavis)
  others.extend(statistics)
  others.extend(resmeth)
  others.extend(proglang)
  others.extend(algmodai)
  others.extend(collect)
  others.extend(sources)
  others.extend(types)
  others.extend(analy)
  others.extend(applic)
  others.extend(sim)
  others.extend(software)
  others.extend(lab)
  others = list(set(others))
  others.sort()
  
  #remove items from "others" already in "bok"
  #this helps ensure that bok keywords can get a higher weight than non bok keywords 
  i = len(others) - 1
  while i >=0:
    if others[i] in bok:
      del others[i]
    i -=1
  
  
  #stop words definition
  stop_words = list(stopwords.words('english'))
  nlp = spacy.load('en_core_web_sm', disable=['parser', 'ner'])
  stop_words.append('-PRON-')
  first_stops = ['cr','ul','ii','cog','pp','ps','geog','cosc','biol','el','sesp',
                 'eecs','oba','phys','phy','mth','cmsc','nur','ce','cs','iii'] #unkown/unnecessary abbreviations
  second_stops = ['make','impact','apply','change','involve','reside','vary','may',
                  'meet','use','include','pertain','tell','cover','devote',
                  'recognize','carry'] #verbs that are not useful
  third_stops = ['new','minimum','useful','mainly','large','liberal','formerly',
                 'especially','absolutely','graduate','odd','one','throughout',
                 'weekly','least','well','hour','common','require','along','least',
                 'long','related','prior','open','sophomore','junior','single',
                 'necessary'] #unuseful descriptors
  fourth_stops = ['treat','prereq','prerequisite','creditsprerequisite',
                  'corequisite','either','assignment','major','none','arts','core',
                  'andor','semester','hoursprereq','student','instructor','threehour',
                  'within','lecturescover','satisfactoryno','summer','yifat',
                  'givenfor','term','classroom','area','inquiry','researchintensive',
                  'year','via','teacher','ofhow', 'specialstudies', 'research', 'special', 'studies'] #other unuseful words
  stop_words.extend(first_stops)
  stop_words.extend(second_stops)
  stop_words.extend(third_stops)
  stop_words.extend(fourth_stops)
  
  #remove stopwords from df:
  for i in range(len(ds_schools_df)): # for each row
    des = ds_schools_df.loc[i,'Descriptions'].split()
    j = len(des) - 1
    while j >=0:
      if des[j] in stop_words:
        del des[j]
      j-=1
    ds_schools_df.loc[i,'Descriptions'] = ' '.join(des)
  
  #create weights
  ds_schools_df["Weights"] = 0
  for i in range(len(ds_schools_df)): #for each row
    #print(i)
    num = 0
    d = ds_schools_df['Descriptions'][i]
    d = d.split() #split for word count
    denom = len(d) #overall word count
    for b in bok: #for each BoK term
      b = b.split() #split for word count
      b_len = len(b) #num of words in BoK term
      b = ' '.join(b) #join for analysis
      d = ' '.join(d).lower() #join for analysis
      token = nltk.word_tokenize(d) #tokenize for ngrams
      ngram = list(ngrams(token,b_len)) #create ngrams
      for n in range(len(ngram)): #turn tuples to strings
        ngram[n] = ' '.join(ngram[n])
      num += (ngram.count(b) * b_len)
      d = d.split() #split for continuity
    for b in others: #for each "others" term
      b = b.split() #split for word count
      b_len = len(b) #num of words in "others" term
      b = ' '.join(b) #join for analysis
      d = ' '.join(d) #join for analysis
      token = nltk.word_tokenize(d) #tokenize for ngrams
      ngram = list(ngrams(token,b_len)) #create ngrams
      for n in range(len(ngram)): #turn tuples to strings
        ngram[n] = ' '.join(ngram[n])
      num += (ngram.count(b) * 0.5)
      d = d.split() #split for continuity
    frac = float(num/denom)
    ds_schools_df.loc[i, 'Weights'] = frac
  print("Creating 'csvs/0713_bok_courses_cat_weights.csv'...")
  ds_schools_df.to_csv('csvs/0713_bok_courses_cat_weights.csv',index=False)
  
  #process words, create dictionaries for future function calls
  responses, ds_school_list = process_words(stop_words,ds_schools_df)
  
  #tfidf analysis
  tfidf(responses,ds_school_list)
  
  
  #MACHINE lEARNING
  #make floats finite
  for i in range(len(ds_schools_df)):
    ds_schools_df.loc[i, 'Weights'] = float('%.5f'%(ds_schools_df.loc[i,'Weights']))
  
  #create gower matrix & linkage
  dm = gower.gower_matrix(ds_schools_df)
  
  # determine k using elbow method
  
  x1 = np.array(ds_schools_df['Sum'])
  x2 = np.array(ds_schools_df['Weights'])
  
  #plot points
  plt.plot()
  plt.xlim([0, 20])
  plt.ylim([0, 1])
  plt.title('Dataset')
  plt.scatter(x1, x2)
  plt.show()
  
  # create new plot and data
  plt.plot()
  X = np.array(list(zip(x1, x2))).reshape(len(x1), 2)
  colors = ['b', 'g', 'r']
  markers = ['o', 'v', 's']
  
  # k means determine k
  distortions = []
  K = range(1,10)
  for k in K:
      kmeanModel = KMeans(n_clusters=k).fit(X)
      kmeanModel.fit(X)
      distortions.append(sum(np.min(cdist(X, kmeanModel.cluster_centers_, 'euclidean'), axis=1)) / X.shape[0])
  
  # Plot the elbow
  plt.plot(K, distortions, 'bx-')
  plt.xlabel('k')
  plt.ylabel('Distortion')
  plt.title('The Elbow Method showing the optimal k')
  plt.show()
  
  # create kmeans object
  kmeans = KMeans(n_clusters=4)
  # fit kmeans object to data
  kmeans.fit(dm)
  
  # save new clusters for chart
  y_km = kmeans.fit_predict(dm)
  ds_schools_df['category']=pd.DataFrame(y_km)
  
  groups = ds_schools_df.groupby("category")
  for name, group in groups:
      plt.plot(group["Weights"], group["Sum"], marker="o", linestyle="", label=name)
  plt.legend()
  
  #save to csv
  print("Creating 'csvs/SMITH_FINAL_THE_results.csv'...")
  ds_schools_df.to_csv('csvs/SMITH_FINAL_THE_results.csv', index=False)
  
  
  
apis: pandas.read_csv
apis: matplotlib.pyplot.legend scipy.spatial.distance.cdist pandas.read_csv pandas.DataFrame matplotlib.pyplot.xlim matplotlib.pyplot.title sklearn.cluster.KMeans matplotlib.pyplot.show matplotlib.pyplot.ylabel matplotlib.pyplot.ylim numpy.array matplotlib.pyplot.plot matplotlib.pyplot.xlabel matplotlib.pyplot.scatter