Deep learning chatbot specific Index error list index out of range
I am trying to follow a tutorial on how to make a deeplearning chatbot with pytorch. However, this code is quite complex for me and it has stopped with a &quot;IndexError: list index out of range&quot;. I looked the error up and get the gist of what it usually means, but seeing as this code is very complex for me I can't figure out how to solve the error.
this is the source tutorial: [https://colab.research.google.com/github/pytorch/tutorials/blob/gh-pages/_downloads/chatbot_tutorial.ipynb#scrollTo=LTzdbPF-OBL9][1]
Line 198 seems to be causing the error
return len(p[0].split(' ')) &lt; MAX_LENGTH and len(p[1].split(' ')) &lt; MAX_LENGTH

This is the error log
Start preparing training data ...
Reading lines...
Traceback (most recent call last):
  File &quot;D:\Documents\Python\python pycharm files\pythonProject4\3.9 Chatbot.py&quot;, line 221, in &lt;module&gt;
    voc, pairs = loadPrepareData(corpus, corpus_name, datafile, save_dir)
  File &quot;D:\Documents\Python\python pycharm files\pythonProject4\3.9 Chatbot.py&quot;, line 209, in loadPrepareData
    pairs = filterPairs(pairs)
  File &quot;D:\Documents\Python\python pycharm files\pythonProject4\3.9 Chatbot.py&quot;, line 202, in filterPairs
    return [pair for pair in pairs if filterPair(pair)]
  File &quot;D:\Documents\Python\python pycharm files\pythonProject4\3.9 Chatbot.py&quot;, line 202, in &lt;listcomp&gt;
    return [pair for pair in pairs if filterPair(pair)]
  File &quot;D:\Documents\Python\python pycharm files\pythonProject4\3.9 Chatbot.py&quot;, line 198, in filterPair
    return len(p[0].split(' ')) &lt; MAX_LENGTH and len(p[1].split(' ')) &lt; MAX_LENGTH
IndexError: list index out of range
Read 442563 sentence pairs

Process finished with exit code 1

And this is my code copied from my pycharm up to the block with the error. Seeing as its a huge code I could not copy the entire code. The rest of the code can be found in the github source link above.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import torch
from torch.jit import script, trace
import torch.nn as nn
from torch import optim
import torch.nn.functional as F
import csv
import random
import re
import os
import unicodedata
import codecs
from io import open
import itertools
import math


USE_CUDA = torch.cuda.is_available()
device = torch.device(&quot;cuda&quot; if USE_CUDA else &quot;cpu&quot;)

corpus_name = &quot;cornell movie-dialogs corpus&quot;
corpus = os.path.join(&quot;D:\Documents\Python\intents&quot;, corpus_name)

def printLines(file, n=10):
    with open(file, 'rb') as datafile:
        lines = datafile.readlines()
    for line in lines[:n]:
        print(line)

printLines(os.path.join(corpus, &quot;movie_lines.txt&quot;))

# Splits each line of the file into a dictionary of fields
def loadLines(fileName, fields):
    lines = {}
    with open(fileName, 'r', encoding='iso-8859-1') as f:
        for line in f:
            values = line.split(&quot; +++$+++ &quot;)
            # Extract fields
            lineObj = {}
            for i, field in enumerate(fields):
                lineObj[field] = values[i]
            lines[lineObj['lineID']] = lineObj
    return lines


# Groups fields of lines from `loadLines` into conversations based on *movie_conversations.txt*
def loadConversations(fileName, lines, fields):
    conversations = []
    with open(fileName, 'r', encoding='iso-8859-1') as f:
        for line in f:
            values = line.split(&quot; +++$+++ &quot;)
            # Extract fields
            convObj = {}
            for i, field in enumerate(fields):
                convObj[field] = values[i]
            # Convert string to list (convObj[&quot;utteranceIDs&quot;] == &quot;['L598485', 'L598486', ...]&quot;)
            lineIds = eval(convObj[&quot;utteranceIDs&quot;])
            # Reassemble lines
            convObj[&quot;lines&quot;] = []
            for lineId in lineIds:
                convObj[&quot;lines&quot;].append(lines[lineId])
            conversations.append(convObj)
    return conversations


# Extracts pairs of sentences from conversations
def extractSentencePairs(conversations):
    qa_pairs = []
    for conversation in conversations:
        # Iterate over all the lines of the conversation
        for i in range(len(conversation[&quot;lines&quot;]) - 1):  # We ignore the last line (no answer for it)
            inputLine = conversation[&quot;lines&quot;][i][&quot;text&quot;].strip()
            targetLine = conversation[&quot;lines&quot;][i+1][&quot;text&quot;].strip()
            # Filter wrong samples (if one of the lists is empty)
            if inputLine and targetLine:
                qa_pairs.append([inputLine, targetLine])
    return qa_pairs

# Define path to new file
datafile = os.path.join(corpus, &quot;formatted_movie_lines.txt&quot;)

delimiter = '\t'
# Unescape the delimiter
delimiter = str(codecs.decode(delimiter, &quot;unicode_escape&quot;))

# Initialize lines dict, conversations list, and field ids
lines = {}
conversations = []
MOVIE_LINES_FIELDS = [&quot;lineID&quot;, &quot;characterID&quot;, &quot;movieID&quot;, &quot;character&quot;, &quot;text&quot;]
MOVIE_CONVERSATIONS_FIELDS = [&quot;character1ID&quot;, &quot;character2ID&quot;, &quot;movieID&quot;, &quot;utteranceIDs&quot;]

# Load lines and process conversations
print(&quot;\nProcessing corpus...&quot;)
lines = loadLines(os.path.join(corpus, &quot;movie_lines.txt&quot;), MOVIE_LINES_FIELDS)
print(&quot;\nLoading conversations...&quot;)
conversations = loadConversations(os.path.join(corpus, &quot;movie_conversations.txt&quot;),
                                  lines, MOVIE_CONVERSATIONS_FIELDS)

# Write new csv file
print(&quot;\nWriting newly formatted file...&quot;)
with open(datafile, 'w', encoding='utf-8') as outputfile:
    writer = csv.writer(outputfile, delimiter=delimiter)
    for pair in extractSentencePairs(conversations):
        writer.writerow(pair)

# Print a sample of lines
print(&quot;\nSample lines from file:&quot;)
printLines(datafile)

# Default word tokens
PAD_token = 0  # Used for padding short sentences
SOS_token = 1  # Start-of-sentence token
EOS_token = 2  # End-of-sentence token

class Voc:
    def __init__(self, name):
        self.name = name
        self.trimmed = False
        self.word2index = {}
        self.word2count = {}
        self.index2word = {PAD_token: &quot;PAD&quot;, SOS_token: &quot;SOS&quot;, EOS_token: &quot;EOS&quot;}
        self.num_words = 3  # Count SOS, EOS, PAD

    def addSentence(self, sentence):
        for word in sentence.split(' '):
            self.addWord(word)

    def addWord(self, word):
        if word not in self.word2index:
            self.word2index[word] = self.num_words
            self.word2count[word] = 1
            self.index2word[self.num_words] = word
            self.num_words += 1
        else:
            self.word2count[word] += 1

    # Remove words below a certain count threshold
    def trim(self, min_count):
        if self.trimmed:
            return
        self.trimmed = True

        keep_words = []

        for k, v in self.word2count.items():
            if v &gt;= min_count:
                keep_words.append(k)

        print('keep_words {} / {} = {:.4f}'.format(
            len(keep_words), len(self.word2index), len(keep_words) / len(self.word2index)
        ))

        # Reinitialize dictionaries
        self.word2index = {}
        self.word2count = {}
        self.index2word = {PAD_token: &quot;PAD&quot;, SOS_token: &quot;SOS&quot;, EOS_token: &quot;EOS&quot;}
        self.num_words = 3 # Count default tokens

        for word in keep_words:
            self.addWord(word)

MAX_LENGTH = 10  # Maximum sentence length to consider

# Turn a Unicode string to plain ASCII, thanks to
# http://stackoverflow.com/a/518232/2809427
def unicodeToAscii(s):
    return ''.join(
        c for c in unicodedata.normalize('NFD', s)
        if unicodedata.category(c) != 'Mn'
    )

# Lowercase, trim, and remove non-letter characters
def normalizeString(s):
    s = unicodeToAscii(s.lower().strip())
    s = re.sub(r&quot;([.!?])&quot;, r&quot; \1&quot;, s)
    s = re.sub(r&quot;[^a-zA-Z.!?]+&quot;, r&quot; &quot;, s)
    s = re.sub(r&quot;\s+&quot;, r&quot; &quot;, s).strip()
    return s

# Read query/response pairs and return a voc object
def readVocs(datafile, corpus_name):
    print(&quot;Reading lines...&quot;)
    # Read the file and split into lines
    lines = open(datafile, encoding='utf-8').\
        read().strip().split('\n')
    # Split every line into pairs and normalize
    pairs = [[normalizeString(s) for s in l.split('\t')] for l in lines]
    voc = Voc(corpus_name)
    return voc, pairs

# Returns True iff both sentences in a pair 'p' are under the MAX_LENGTH threshold
def filterPair(p):
    # Input sequences need to preserve the last word for EOS token
    return len(p[0].split(' ')) &lt; MAX_LENGTH and len(p[1].split(' ')) &lt; MAX_LENGTH

# Filter pairs using filterPair condition
def filterPairs(pairs):
    return [pair for pair in pairs if filterPair(pair)]

# Using the functions defined above, return a populated voc object and pairs list
def loadPrepareData(corpus, corpus_name, datafile, save_dir):
    print(&quot;Start preparing training data ...&quot;)
    voc, pairs = readVocs(datafile, corpus_name)
    print(&quot;Read {!s} sentence pairs&quot;.format(len(pairs)))
    pairs = filterPairs(pairs)
    print(&quot;Trimmed to {!s} sentence pairs&quot;.format(len(pairs)))
    print(&quot;Counting words...&quot;)
    for pair in pairs:
        voc.addSentence(pair[0])
        voc.addSentence(pair[1])
    print(&quot;Counted words:&quot;, voc.num_words)
    return voc, pairs


# Load/Assemble voc and pairs
save_dir = os.path.join(&quot;data&quot;, &quot;save&quot;)
voc, pairs = loadPrepareData(corpus, corpus_name, datafile, save_dir)
# Print some pairs to validate
print(&quot;\npairs:&quot;)
for pair in pairs[:10]:
    print(pair)

MIN_COUNT = 3    # Minimum word count threshold for trimming

I really hope someone can help me fix this problem and help me understand why it happens.

In the end I changed
return len(p[0].split(' ')) &lt; MAX_LENGTH and len(p[1].split(' ')) &lt; MAX_LENGTH

to
try:
    return len(p[0].split(' ')) &lt; MAX_LENGTH and len(p[1].split(' ')) &lt; MAX_LENGTH
except:
    return False

And now the code seems to be working.
