attributes.addElement(new Attribute(word));
attributes.addElement(new Attribute(m_Prefix + word));
  private void determineDictionary() {        // System.err.println("Creating dictionary");         int classInd = getInputFormat().classIndex();    int values = 1;    if (classInd != -1) {      values = getInputFormat().attribute(classInd).numValues();    }    TreeMap dictionaryArr [] = new TreeMap[values];    for (int i = 0; i < values; i++) {      dictionaryArr[i] = new TreeMap();    }    // Make sure we know which fields to convert    determineSelectedRange();    // Tokenize all training text into an orderedMap of "words".    for (int i = 0; i < getInputFormat().numInstances(); i++) {      /*	if (i % 10 == 0) {        System.err.print( i + " " + getInputFormat().numInstances() + "\r");         System.err.flush();	}      */      Instance instance = getInputFormat().instance(i);      for (int j = 0; j < instance.numAttributes(); j++) {         if (m_SelectedRange.isInRange(j) && (instance.isMissing(j) == false)) {	  //getInputFormat().attribute(j).type() == Attribute.STRING                       StringTokenizer st = new StringTokenizer(instance.stringValue(j),                                                   delimiters);          while (st.hasMoreTokens()) {            String word = st.nextToken().intern();            int vInd = 0;            if (classInd != -1) {              vInd = (int)instance.classValue();            }            Count count = (Count)dictionaryArr[vInd].get(word);            if (count == null) {              dictionaryArr[vInd].put(word, new Count(1));            } else {              count.count ++;            }          }        }      }    }    int totalsize = 0;    int prune[] = new int[values];    for (int z = 0; z < values; z++) {      totalsize += dictionaryArr[z].size();      int array[] = new int[dictionaryArr[z].size()];      int pos = 0;      Iterator it = dictionaryArr[z].keySet().iterator();      while (it.hasNext()) {        String word = (String)it.next();        Count count = (Count)dictionaryArr[z].get(word);        array[pos] = count.count;        pos++;      }      // sort the array      sortArray(array);      if (array.length < m_WordsToKeep) {        // if there aren't enough words, set the threshold to 1        prune[z] = 1;      } else {        // otherwise set it to be at least 1        prune[z] = Math.max(1, array[array.length - m_WordsToKeep]);      }    }    /*      for (int z=0;z<values;z++) {      System.err.println(dictionaryArr[z].size()+" "+totalsize);      }    */    // Convert the dictionary into an attribute index    // and create one attribute per word    FastVector attributes = new FastVector(totalsize +					   getInputFormat().numAttributes());    // Add the non-converted attributes     int classIndex = -1;    for (int i = 0; i < getInputFormat().numAttributes(); i++) {      if (!m_SelectedRange.isInRange(i)) {         if (getInputFormat().classIndex() == i) {          classIndex = attributes.size();        }	attributes.addElement(getInputFormat().attribute(i).copy());      }         }    // Add the word vector attributes    TreeMap newDictionary = new TreeMap();    int index = attributes.size();    for(int z = 0; z < values; z++) {      /*	System.err.print("\nCreating word index...");	if (values > 1) {        System.err.print(" for class id=" + z); 	}	System.err.flush();      */      Iterator it = dictionaryArr[z].keySet().iterator();      while (it.hasNext()) {        String word = (String)it.next();        Count count = (Count)dictionaryArr[z].get(word);        if (count.count >= prune[z]) {          //          System.err.println(word+" "+newDictionary.get(word));          if(newDictionary.get(word) == null) {            /*	      if (values > 1) {              System.err.print(getInputFormat().classAttribute().value(z) + " ");	      }	      System.err.println(word);            */            newDictionary.put(word, new Integer(index++));            attributes.addElement(new Attribute(word));          }        }      }    }    attributes.trimToSize();    m_Dictionary = newDictionary;    //System.err.println("done: " + index + " words in total.");        // Set the filter's output format    Instances outputFormat = new Instances(getInputFormat().relationName(),                                            attributes, 0);    outputFormat.setClassIndex(classIndex);    setOutputFormat(outputFormat);  }