code: package zemberek.corpus;
  
  import java.io.IOException;
  import java.nio.file.Files;
  import java.nio.file.Path;
  import java.nio.file.Paths;
  import java.util.Comparator;
  import java.util.List;
  import java.util.stream.Collectors;
  import zemberek.core.logging.Log;
  
  public class EliminateDuplicates {
  
    private Path corporaRoot;
  
    private EliminateDuplicates(Path corporaRoot) {
      this.corporaRoot = corporaRoot;
    }
  
    public static void main(String[] args) throws IOException {
      Path cr = Paths.get("/media/data/corpora/raw3");
      EliminateDuplicates e = new EliminateDuplicates(cr);
      e.doIt();
    }
  
    private void doIt() throws IOException {
      List<Path> dirs = Files.walk(corporaRoot)
          .filter(s -> s.toFile().isDirectory()).collect(Collectors.toList());
      for (Path dir : dirs) {
        if (dir.equals(corporaRoot)) {
          continue;
        }
        extractLabeledDocuments(dir, corporaRoot.resolve(dir.toFile().getName() + ".corpus"));
      }
    }
  
    private void extractLabeledDocuments(Path root, Path outFile) throws IOException {
      List<Path> files = Files.walk(root).filter(s -> s.toFile().isFile())
          .collect(Collectors.toList());
      files.sort(Comparator.comparing(Path::toString));
      WebCorpus corpus = new WebCorpus("c", "c");
      for (Path file : files) {
        if (file.toFile().isDirectory()) {
          continue;
        }
        Log.info("Adding %s", file);
        List<WebDocument> doc = WebCorpus.loadDocuments(file);
        for (WebDocument webDocument : doc) {
          webDocument.removeDuplicateLines();
        }
        List<WebDocument> labeled = doc.stream()
            .filter(s -> s.getContentAsString().length() > 200)
            .collect(Collectors.toList());
        corpus.addDocuments(labeled);
      }
      Log.info("Total amount of files = %d", corpus.getDocuments().size());
      WebCorpus noDuplicates = corpus.copyNoDuplicates();
      Log.info("Corpus size = %d, After removing duplicates = %d",
          corpus.documentCount(),
          noDuplicates.documentCount());
      Log.info("Saving corpus to %s", outFile);
      noDuplicates.save(outFile, false);
    }
  }
  