code: package com.hrLDA.main;
  
  import java.io.*;
  import java.util.*;
  import java.util.regex.*;
  
  import cc.mallet.pipe.*;//TokenSequence2FeatureSequenceWithBigrams
  import cc.mallet.pipe.iterator.*;
  import cc.mallet.types.*;//InstanceList
  
  public class ImportDir {
  
      Pipe pipe;
      String endString;
  
      public ImportDir(String endString) {
          this.endString=endString;
          pipe = buildPipe();
      }
  
      public Pipe buildPipe() {
          ArrayList <Pipe> pipeList = new ArrayList <Pipe>();
  
          // Read data from File objects
          pipeList.add(new Input2CharSequenceFilter("UTF-8"));
  //        pipeList.add( new CharSequenceLowercase() );
          pipeList.add( new CharSequence2TokenSequence(Pattern.compile("\\p{L}[\\p{L}\\p{P}]+\\p{L}")) );
          pipeList.add( new TokenSequenceRemoveStopwords(new File("stoplists/en.txt"), "UTF-8", false, false, false) );
          pipeList.add( new TokenSequence2FeatureSequence() );
  
          
  
          /** Regular expression for what constitutes a token.
          *This pattern includes Unicode letters, Unicode numbers, 
          *and the underscore character. Alternatives:
          *"\\S+"   (anything not whitespace)
          *"\\w+"    ( A-Z, a-z, 0-9, _ )
          *"[\\p{L}\\p{N}_]+|[\\p{P}]+"   (a group of only letters and numbers OR
          *a group of only punctuation marks)
          */        
  //        Pattern tokenPattern =
  //                Pattern.compile("\\p{L}[\\p{L}\\p{P}]+\\p{L}");        
  //
  //        // Tokenize raw strings
  //        pipeList.add(new CharSequence2TokenSequence(tokenPattern));
  
          pipeList.add(new TokenSequenceRemoveNonAlpha());
          
          // Normalize all tokens to all lowercase
          pipeList.add(new TokenSequenceLowercase());
  
          // Remove stopwords from a standard English stoplist.
          //  options: [case sensitive] [mark deletions]
          pipeList.add(new TokenSequenceRemoveStopwords(new File("stoplists/en.txt"), 
          		"UTF-8", false, false, false) );
  
  
          return new SerialPipes(pipeList);
      }
  
      public InstanceList readDirectory(File directory) {
          return readDirectories(new File[] {directory});
      }
  
      public InstanceList readDirectories(File[] directories) {
          
          // Construct a file iterator, starting with the 
          //  specified directories, and recursing through subdirectories.
          // The second argument specifies a FileFilter to use to select
          //  files within a directory.
          // The third argument is a Pattern that is applied to the 
          //   filename to produce a class label. In this case, I've 
          //   asked it to use the last directory name in the path.
          FileIterator iterator =
              new FileIterator(directories,
                               new TxtFilter(),
                               FileIterator.LAST_DIRECTORY);
          // Construct a new instance list, passing it the pipe
          //  we want to use to process instances.
          InstanceList instances = new InstanceList(pipe);
  
          // Now process each instance provided by the iterator.
          instances.addThruPipe(iterator);
  
          return instances;
      }
  
      /** This class illustrates how to build a simple file filter */
      class TxtFilter implements FileFilter {
  
          /** Test whether the string representation of the file 
           *   ends with the correct extension. Note that {@ref FileIterator}
           *   will only call this filter if the file is not a directory,
           *   so we do not need to test that it is a file.
           */
          public boolean accept(File file) {
              return file.toString().endsWith(".txt");
          }
      }
  
  }