# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Notebook for testing performance of NLC classification
# [Watson Developer Cloud](https://www.ibm.com/watsondevelopercloud) is a platform of cognitive services that leverage machine learning techniques to help partners and clients solve a variety business problems. Furthermore, several of the WDC services fall under the **supervised learning** suite of machine learning algorithms, that is, algorithms that learn by example. This begs the questions: "How many examples should we provide?" and "When is my solution ready for prime time?"
#
# It is critical to understand that training a machine learning solution is an iterative process where it is important to continually improve the solution by providing new examples and measuring the performance of the trained solution. In this notebook, we show how you can compute important Machine Learning metrics (accuracy, precision, recall, confusion_matrix) to judge the performance of your solution. For more details on these various metrics, please consult the **[Is Your Chatbot Ready for Prime-Time?](https://developer.ibm.com/dwblog/2016/chatbot-cognitive-performance-metrics-accuracy-precision-recall-confusion-matrix/)** blog.
#

# <br> The notebook assumes you have already created a Watson [Natural Language Classifier](https://www.ibm.com/watson/developercloud/nl-classifier.html) instance and trained a classifier. </br>
# <br> To leverage this notebook, you need to provide the following information</br>
# * Credentials for your NLC instance (username and password)
# * id for your trained classifier (this is returned when you train an NLC classifier)
# * csv file with your text utterances and corresponding class labels
# * results csv file to write the results to

# Only run this cell if you don't have pandas_ml or watson_developer_cloud installed
# !pip install pandas_ml
# You can specify the latest verion of watson_developer_cloud (1.0.0 as of November 20, 2017)
# !pip install -I watson-developer-cloud==1.0.0

#Import utilities
import json
import sys
import codecs
import unicodecsv as csv
from sklearn.metrics import confusion_matrix
from sklearn.metrics import precision_recall_fscore_support
from sklearn.metrics import classification_report
from sklearn.metrics import accuracy_score
import matplotlib.pyplot as plt
import pandas_ml
from pandas_ml import ConfusionMatrix
from watson_developer_cloud import NaturalLanguageClassifierV1

# Provide the path to the parms file which includes credentials to access your NLC service as well as the input
# test csv file and the output csv file to write the output results to.

# Sample parms file data
#{
#	"url": "https://gateway.watsonplatform.net/natural-language-classifier/api/v1/classifiers",
#	"user":"YOUR_NLC_USERNAME",
#	"password": "<PASSWORD>",
#	"nlc_id":"YOUR_NLC_CLASSIFIER_ID",
#	"test_csv_file": "COMPLETE_PATH_TO_YOUR_TEST_CSV_FILE",
#	"results_csv_file": "COMPLETE PATH TO RESULTS FILE (any file you can write to)",
#	"confmatrix_csv_file": "COMPLETE PATH TO CONFUSION MATRIX FILE (any file you can write to)"
#}


# +
# Provide complete path to the file which includes all required parms
# A sample parms file is included (example_parms.json)
nlcParmsFile = 'COMPLETE PATH TO YOUR PARMS FILE'
parms = ''
with open(nlcParmsFile) as parmFile:
    parms = json.load(parmFile)

url=parms['url']
user=parms['user']
password=parms['password']
nlc_id=parms['nlc_id']
test_csv_file=parms['test_csv_file']
results_csv_file=parms['results_csv_file']
confmatrix_csv_file=parms['confmatrix_csv_file']

json.dumps(parms)

# Create an object for your NLC instance
natural_language_classifier = NaturalLanguageClassifierV1(
  username=user,
  password=password)


# -

# Define useful methods to classify using trained NLC classifier.

# +
# Given a text string and a pointer to NLC instance and classifierID, get back NLC response
def getNLCresponse(nlc_instance,classifierID,string):
    # remove newlines from input text as that causes WCS to return an error 
    string = string.replace("\n","")
    classes = nlc_instance.classify(classifierID, string)
    return classes

# Process multiple text utterances (provided via csv file) in batch. Effectively, read the csv file and for each text
# utterance, get NLC response. Aggregate and return results.
def batchNLC(nlc_instance,classifierID,csvfile):
    test_classes=[]
    nlcpredict_classes=[]
    nlcpredict_confidence=[]
    text=[]
    i=0
    print ('reading csv file: ', csvfile)
    with open(csvfile, 'rb') as csvfile:
        # For better handling of utf8 encoded text
        csvReader = csv.reader(csvfile, encoding="utf-8-sig")
        for row in csvReader:
            print(row)
            # Assume input text is 2 column csv file, first column is text
            # and second column is the label/class/intent
            # Sometimes, the text string includes commas which may split
            # the text across multiple colmns. The following code handles that.
            if len(row) > 2:
                qelements = row[0:len(row)-1]
                utterance = ",".join(qelements)
                test_classes.append(row[len(row)-1])
            else:
                utterance = row[0]
                test_classes.append(row[1])
            utterance = utterance.replace('\r', ' ')
            print ('i: ', i, ' testing row: ', utterance)
            
            #test_classes.append(row['class'])
            #print 'analyzing row: ', i, ' text: ', row['text']
            nlc_response = getNLCresponse(nlc_instance,classifierID,utterance)
            if nlc_response['classes']:
                nlcpredict_classes.append(nlc_response['classes'][0]['class_name'])
                nlcpredict_confidence.append(nlc_response['classes'][0]['confidence'])
            else:
                nlcpredict_classes.append('')
                nlcpredict_confidence.append(0)
            text.append(utterance)
            
            i = i+1
            if(i%250 == 0):
                print("")
                print("Processed ", i, " records")
            if(i%10 == 0):
                sys.stdout.write('.')
        print("")
        print("Finished processing ", i, " records")
    return test_classes, nlcpredict_classes, nlcpredict_confidence, text

# Plot confusion matrix as an image
def plot_conf_matrix(conf_matrix):
    plt.figure()
    plt.imshow(conf_matrix)
    plt.show()

# Print confusion matrix to a csv file
def confmatrix2csv(conf_matrix,labels,csvfile):
    with open(csvfile, 'wb') as csvfile:
        csvWriter = csv.writer(csvfile)
        row=list(labels)
        row.insert(0,"")
        csvWriter.writerow(row)
        for i in range(conf_matrix.shape[0]):
            row=list(conf_matrix[i])
            row.insert(0,labels[i])
            csvWriter.writerow(row)
            


# +
# This is an optional step to quickly test response from NLC for a given utterance
#testQ='can I reset my password'
#results = getNLCresponse(natural_language_classifier,nlc_id,testQ)
#print(json.dumps(results, indent=2))
# -

# Call NLC on the specified csv file and collect results.

test_classes,nlcpredict_classes,nlcpredict_conf,text=batchNLC(natural_language_classifier,nlc_id,test_csv_file)

# print results to csv file including original text, the correct label, 
# the predicted label and the confidence reported by NLC.
csvfileOut=results_csv_file
with open(csvfileOut, 'wb') as csvOut:
    outrow=['text','true class','NLC Predicted class','Confidence']
    csvWriter = csv.writer(csvOut,dialect='excel')
    csvWriter.writerow(outrow)
    for i in range(len(text)):
        outrow=[text[i],test_classes[i],nlcpredict_classes[i],str(nlcpredict_conf[i])]
        csvWriter.writerow(outrow)

# Compute confusion matrix
labels=list(set(test_classes))
nlc_confusion_matrix = confusion_matrix(test_classes, nlcpredict_classes, labels)
nlcConfMatrix = ConfusionMatrix(test_classes, nlcpredict_classes)

# Print out confusion matrix with labels to csv file
confmatrix2csv(nlc_confusion_matrix,labels,confmatrix_csv_file)

# %matplotlib inline
nlcConfMatrix.plot()

# Compute accuracy of classification
acc=accuracy_score(test_classes, nlcpredict_classes)
print('Classification Accuracy: ', acc)

# print precision, recall and f1-scores for the different classes
print(classification_report(test_classes, nlcpredict_classes, labels=labels))

# +
#Optional if you would like each of these metrics separately
#[precision,recall,fscore,support]=precision_recall_fscore_support(test_classes, nlcpredict_classes, labels=labels)
#print("precision: ", precision)
#print("recall: ", recall)
#print("f1 score: ", fscore)
#print("support: ", support)

StarcoderdataJupyterScriptsDedupFiltered