I've been working on a project where I am importing a large document of documents and tokenizing each document. I then make a hashset of the tokens I have, and for each of these unique tokens I am looking to find their frequency in each of the documents I have. There are about 130000 documents total.

I've run the code and unfortunately, it's taking 150 hrs to run. Do you have any suggestions on improving my code?

This is my `Tokenizer()` function, which surprisingly works okay:

```
static private IEnumerable<string> Tokenizer(ref StreamReader sr, ref List<string> tokenz, ref List<string> stopwords)//function for tokenizing
{
    string line;
    var comparer = StringComparer.InvariantCultureIgnoreCase;
    var StopWordSet = new HashSet<string>(stopwords, comparer);
    List<string> tokens = new List<string>();//list of strings called tokens
    while ((line = sr.ReadLine()) != null)//as long as the streamreader has something
    {
        foreach (string item in line.Split(' '))//split amongst strings
        {
            if (item.StartsWith("<") & item.EndsWith(">"))
            {
                item.Trim();//trims the item of spaces
                if (item == "</DOC>")
                {
                    //return item;
                    tokenz.Add(item);//adds the doc tags for later separation use
                }
            }
            else
            {
                string newitem;
                item.Trim();//trims the item of spaces
                if (item != "")//ensures item is not blank
                {
                    newitem = Regex.Replace(item, @"[^A-Za-z0-9]+", "", RegexOptions.IgnoreCase);//regex allows us to ignore case and remove any special characters
                    string newitem2 = newitem.ToLower();
                    {
                        if (StopWordSet.Contains(newitem2))
                        {
                        }
                        else
                        {
                            tokenz.Add(newitem2);
                        }
                        //tokens.Add(newitem.ToLower());//makes all lower case
                    }
                }
            }
        }
    }
    return tokens;
}

```

The real issue is here:

```
static public void AddToDictionaryAndCount(ref int doccounter2, ref HashSet<string> MyLexicon, ref List<string> tokens, ref Dictionary<int, int> DocFreqCounter, ref Dictionary<string, Dictionary<int, int>> MyDictionary, ref Dictionary<int, int> DocWordCounter)
{
    foreach (string item in MyLexicon)
    {
        int counter = 0;
        int secondcounter = 0;
        int doccounter = 1;
        int termcounter = 0;
        while (counter <= tokens.LastIndexOf("</DOC>"))
        {
            if (tokens[counter] == "</DOC>")
            {
                DocFreqCounter.Add(doccounter, termcounter);
                if (doccounter2 < doccounter)
                {
                    DocWordCounter.Add(doccounter, (counter - secondcounter));
                    doccounter2++;
                }
                termcounter = 0;
                secondcounter = counter;
                doccounter++;
            }
            if (tokens[counter] == item)
            {
                termcounter++;
                //words.termCount = termcounter;
            }
            counter++;
        }
        MyDictionary.Add(item, new Dictionary<int, int>(DocFreqCounter));
        DocFreqCounter.Clear();
    }
}

```