# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# +

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

sns.set_theme(style="darkgrid",palette=sns.color_palette("Set2"))
# -

data=pd.read_csv(r"C:\Users\dhmph\Desktop\XBUS-506-01.Visual_Analytics-master\DataScientist.csv")

data = data[['Job Title', 'Job Description', 'Salary Estimate', 'Rating', 'Company Name', 'Location', 'Industry','Sector']]

data

data['Job Title'].value_counts().head(10)

data['Industry'].value_counts().head(10)

data['Sector'].value_counts().head(10)

data['Location'].nunique()

data['Location'].value_counts().head(10)

df_location=data['Location'].value_counts().head(10)

df_location

austin_data = data.loc[data['Location'] == 'Austin, TX']

chicago_data = data.loc[data['Location'] == 'Chicago, IL']

import nltk
import re
from string import digits
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from nltk.stem import PorterStemmer
from nltk.tokenize import RegexpTokenizer
from nltk.stem import WordNetLemmatizer
ps = PorterStemmer()

#making all text in notes column lowercase
data["Job Description"]=data["Job Description"].str.lower()
data

tokenizer = RegexpTokenizer(r'\w+')
data["Job Description"] = data["Job Description"].apply(lambda x: tokenizer.tokenize(x.lower()))
data["Job Description"]

# +
lemmatizer = WordNetLemmatizer()

def word_lemmatizer(text):
    lem_text = [lemmatizer.lemmatize(i) for i in text]
    return lem_text


# -

data["Job Description"] = data["Job Description"].apply(lambda x: word_lemmatizer(x))
data


def remove_stopwords(text):
    words = [w for w in text if w not in stopwords.words('english')]
    return words


# +

data["Job Description"] = data["Job Description"].apply(lambda x: remove_stopwords(x))
# -

data

austin_data = data.loc[data['Location'] == 'Austin, TX']
chicago_data = data.loc[data['Location'] == 'Chicago, IL']

austin_data



def common_data_science_words(text):
    common_data_science_words = ['data', 'science', 'work', 'experience', 'business','team', 'information', 'employee', 'technology', 'year',
'analysis',
'application',
'system',
'service',
'ability',
'job',
'opportunity',
'development',
'solution',
'design',
'product',
'customer',
'required',
'engineering',
'skill',
'including',
'analytics',
'project',
'requirement',
'learning',
'tool',
'technical',
'support',
'software',
'company',
'position',
'employment']
    words =[w for w in text if w not in common_data_science_words]
    return words

austin_data["Job Description"] = austin_data["Job Description"].apply(lambda x: common_data_science_words(x))

chicago_data["Job Description"] = chicago_data["Job Description"].apply(lambda x: common_data_science_words(x))

# # topic modeling 

import gensim
from gensim.utils import simple_preprocess
import gensim.corpora as corpora
from pprint import pprint


data_words_austin = list(austin_data['Job Description'])

# ### Austin LDA

# Create Dictionary
id2word = corpora.Dictionary(data_words_austin)
# Create Corpus
texts = data_words_austin
# Term Document Frequency
corpus = [id2word.doc2bow(text) for text in texts]
# LDA model training
# number of topics
num_topics = 10
# Build LDA model
lda_model_austin = gensim.models.LdaMulticore(corpus=corpus,
                                       id2word=id2word,
                                       num_topics=num_topics)
# Print the Keyword in the 20 topics
pprint(lda_model_austin.print_topics())
doc_lda = lda_model_austin[corpus]

# ### Chicago LDA

# Create Dictionary
id2word = corpora.Dictionary(data_words_chicago)
# Create Corpus
texts = data_words_chicago
# Term Document Frequency
corpus = [id2word.doc2bow(text) for text in texts]
# LDA model training
# number of topics
num_topics = 10
# Build LDA model
lda_model_chicago = gensim.models.LdaMulticore(corpus=corpus,
                                       id2word=id2word,
                                       num_topics=num_topics)
# Print the Keyword in the 20 topics
pprint(lda_model_chicago.print_topics())
doc_lda = lda_model_chicago[corpus]

# # Load in Topic Modeling visualization

import pyLDAvis
import pyLDAvis.gensim_models as gensimvis
# Visualize the topics
pyLDAvis.enable_notebook()

vis = pyLDAvis.gensim_models.prepare(lda_model_austin, corpus, id2word)
vis

vis = pyLDAvis.gensim_models.prepare(lda_model_chicago, corpus, id2word)
vis



StarcoderdataJupyterScriptsDedupFiltered