# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# # Building offline iPhone spam classifier using coreml
#
# iOS 11 introduced message extension to filter spam messages and coreml to build custom machine learned models to predict spam or not. In this article I’ll go over all the steps I took to build a machine learning model and how I added it to an iphone project.
#
# ## Load Modules

# %matplotlib inline
from io import open
import matplotlib.pyplot as plt
import csv
from textblob import TextBlob
import pandas
import sklearn
import pickle
import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.svm import SVC, LinearSVC
from sklearn.metrics import classification_report, f1_score, accuracy_score, confusion_matrix
from sklearn.pipeline import Pipeline
from sklearn.grid_search import GridSearchCV
from sklearn.cross_validation import StratifiedKFold, cross_val_score, train_test_split 
#from sklearn.model_selection import StratifiedKFold, cross_val_score, train_test_split 
from sklearn.tree import DecisionTreeClassifier 
from sklearn.learning_curve import learning_curve
from nltk.stem.snowball import SnowballStemmer
from nltk.stem.porter import PorterStemmer
from nltk.tokenize import word_tokenize
import string
plt.style.use('ggplot')
pd.options.display.mpl_style = 'default'

# # Data
#
# SMSSpamCollection lets load and understand the data.
#

# +
file_name = 'SMSSpamCollection'
with open(file_name) as f:
    corpus = f.readlines()

print("there are {} messages".format(len(corpus)))
    

corpus = [x.strip() for x in corpus]

for i,message in enumerate(corpus[:10]):
    print(i, message)
# -

# There spam and ham messages are separated by tab. we can use a csv loader to read.

# +
with open('SMSSpamCollection',encoding='UTF-8') as f:
    messages = pd.read_csv(f, sep='\t', quoting=csv.QUOTE_NONE, names=['label', 'message'])

messages['message'] = messages['message'].map(lambda text:text.decode(encoding='utf-8'))
messages.head()
# -

messages.groupby('label').describe()

messages['length'] = messages['message'].map(lambda text:len(text))
messages.head()

messages.length.plot(bins=20, kind='hist')

messages.length.describe()

messages.hist(column='length', by='label', bins=50)

#
# # Processing Data
#
# we need to break the sentences into tokens and stem before we can use it. lets create a method for tokenizing the words.

# +
print("removing punctuations: "+string.punctuation)
stemmer = PorterStemmer()

def tokenize(message):
    """ removes punctuation and tokenizes the words and stems each word.
    """
    msg = "".join([ch for ch in message if ch not in string.punctuation]) # get rid of punctuations
    tokens = word_tokenize(msg) 
    #stems = [stemmer.stem(x).lower() for x in tokens] #correct way to do
    stems = [x.lower() for x in tokens] #iOS does not have porterstemmer, we are going to not use stem for now
    return stems

messages.message.head().apply(tokenize)
# -

# # Feature Vector
#
# we need to convert the tokenized words into a vector to feed into the ml algorithm. we will create a TF-IDF feature vector from the sentence.
#
# 1. create count vector
# 2. convert convert vector into tf-idf vector
#

# +
fv = CountVectorizer(analyzer=tokenize).fit(messages.message)

print("Total number of words in array", len(fv.vocabulary_))
# -

# Lets try to test how different words are represented in the CountVector.

# +
print(fv.transform(["U dun"]))

print("Second:")
print(fv.transform(["dun U"]))

# -

# Notice sentences have same count vector generated. when we fit the vectorizer it creates a vocabulary. this vocabulary is used to create a count vector for all sentences.
#
# Lets try to vectorize a full sentence from our corpus.

print(fv.transform([messages.message[3]]))

#
# ### Count Vector Vocabulary (words_array)
#
# iPhone does not have a CountVectorizer so we would need to do this step if we are to analyze a sentence. we will use the vocabulary of countvectorizer and save into words array file.
#
# words array file is the file with count positions of words and frequency of its occurance.
#
# this is inturn used to calculate term frequency.
#
# tf = Ft / Count(F)
#
# Ft => frequency of term t in current document
# Count(F) => total number of words in corpus. (max of Ft in words array)
#
# lets save this file

import json
with open('words_array.json', 'wb') as fp:
    json.dump(fv.vocabulary_, fp)

# # Messages Feature Vector
#
# Let's compose the feature vector for our entire corpus.

messages_fv = fv.transform(messages.message)
print(messages_fv.shape)

# We will use TF-IDF transformer to transform the count vector of corpus into TFIDF vector.

# +
tfidf = TfidfTransformer().fit(messages_fv)

# test tfidf of same message as before.
t = tfidf.transform(fv.transform([messages.message[3]]))
print(t)
# -

# We see that the values are all normalized and weights are given according to TF-IDF importance. this gives more relevancy for the model to use.
#
# Lets compute the tfidf of the entire corpus.

messages_tfidf = tfidf.transform(messages_fv)
print(messages_tfidf.shape)

# ### IDF 
#
# words idf is simple list of words and their IDF values
#
# idf = log(N/Nt)
#
# N => number of documents
# N => number of documets with word t
#
# we need to compute tfidf of sentence in iOS and we will need the words_idf values as input for computing the vector. lets save the idf array into a file. This is later used in the iOS code.

# +
idf = {}
idf['idf'] = tfidf.idf_.tolist()

with open('words_idf.json', 'wb') as fp:
    json.dump(idf, fp)

print("IDF of corpus :", tfidf.idf_)

# -

# # Model Training
#
# For the model we will use a simple Linear SVM. SVM seems to be getting the most accurate results and we can easily use this model in iPhone as well. lets create and train an Linear SVM model.

# +
# %time spam_detector = LinearSVC().fit(messages_tfidf, messages.label)

predictions = spam_detector.predict(messages_tfidf)

print('accuracy', accuracy_score(messages['label'], predictions))
print('confusion matrix\n', confusion_matrix(messages['label'], predictions))
print('(row=expected, col=predicted)')

# -

# It looks like the model has got a really good accuracy. confusion matrix is also showing great results. 
#
# Lets plot this and view this a little nicely

plt.matshow(confusion_matrix(messages['label'], predictions), cmap=plt.cm.binary, interpolation='nearest')
plt.title('confusion matrix')
plt.colorbar()
plt.ylabel('expected label')
plt.xlabel('predicted label')

print(classification_report(messages['label'], predictions))

# # Convert to CoreML

# +
import coremltools
coreml_model = coremltools.converters.sklearn.convert(spam_detector, "message", "spam_or_not")

#set parameters of the model
coreml_model.short_description = "Classify whether message is spam or not"
coreml_model.input_description["message"] = "TFIDF of message to be classified"
coreml_model.output_description["spam_or_not"] = "Whether message is spam or not"
#save the model
coreml_model.save("SpamMessageClassifier.mlmodel")

StarcoderdataJupyterScriptsDedupFiltered