package zemberek.scratchpad;

import com.google.common.base.Splitter;
import com.google.common.collect.Sets;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import zemberek.core.turkish.Turkish;
import zemberek.morphology.TurkishMorphology;
import zemberek.morphology.lexicon.DictionaryItem;
import zemberek.morphology.lexicon.RootLexicon;

/**
 * Extracts city, district and village names from a csv file. File is downloaded from public postal
 * code file.
 */
public class ExtractTurkishCityDistrictNames {

  static Set<String> avoid = Sets
      .newHashSet("MAH.", "OSB", "OSB.", "MEZRASI", "KÖYÜ", "MERKEZKÖYLER");

  public static void extractSingleWords(Path input, Path output) throws IOException {
    LinkedHashSet<String> provinces = new LinkedHashSet<>();
    LinkedHashSet<String> cities = new LinkedHashSet<>();
    LinkedHashSet<String> districts = new LinkedHashSet<>();
    LinkedHashSet<String> villages = new LinkedHashSet<>();

    List<String> lines = Files.readAllLines(input, StandardCharsets.UTF_8);
    int i = 0;
    for (String line : lines) {
      if (i == 0) { // skip first line.
        i++;
        continue;
      }
      List<String> tokens = Splitter.on('\t').trimResults().omitEmptyStrings().splitToList(line);
      if (tokens.size() != 5) {
        System.out.println("Unexpected data in " + line);
        continue;
      }
      provinces.addAll(clean(tokens.get(0)));
      cities.addAll(clean(tokens.get(1)));
      cities.addAll(clean(tokens.get(2)));
      String districtOrVillage = tokens.get(3);
      if (districtOrVillage.endsWith("MAH.")) {
        districts.addAll(clean(districtOrVillage));
      } else {
        villages.addAll(clean(districtOrVillage));
      }
    }

    try (PrintWriter pw = new PrintWriter(output.toFile(), "utf-8")) {
      pw.println("## --------- Provinces --------");
      provinces.forEach(pw::println);
      pw.println("## --------- Cities    --------");
      cities.forEach(pw::println);
      pw.println("## --------- Districts --------");
      districts.forEach(pw::println);
      pw.println("## --------- Villages  --------");
      villages.forEach(pw::println);
    }

    LinkedHashSet<String> all = new LinkedHashSet<>();
    all.addAll(provinces);
    all.addAll(cities);
    all.addAll(districts);
    all.addAll(villages);

    try (PrintWriter pw = new PrintWriter(output.toFile() + ".all", "utf-8")) {
      all.stream().sorted(Turkish.STRING_COMPARATOR_ASC).forEach(pw::println);
    }
  }

  public static List<String> clean(String in) {
    in = in.replaceAll("[-()]", " ").replaceAll("^\\.|[/\\\\]", "").trim();
    in = in.replaceAll("OSB ", " OSB ");
    List<String> words = Splitter.on(' ').trimResults().omitEmptyStrings().splitToList(in);
    List<String> result = new ArrayList<>();
    for (final String word : words) {
      if (avoid.stream().anyMatch(s -> s.contains(word))) {
        continue;
      }
      if (word.matches("[\\d ]+|^\\d+.*|")) {
        continue;
      }
      if (word.contains(".") || word.contains("_")) {
        continue;
      }

      result.add(Turkish.capitalize(word));
    }
    return result;
  }

  public static void main(String[] args) throws IOException {
    Path extracted = Paths.get("locations-tr.txt");
    extractSingleWords(
        Paths.get("/media/ahmetaa/depo/pk_2018_08_31.csv"),
        extracted);
    Path exceptZemberek = Paths.get("locations-tr.dict");
    removeZemberekDictionaryWordsFromList(extracted, exceptZemberek);
  }

  private static void removeZemberekDictionaryWordsFromList(Path input, Path out)
      throws IOException {
    LinkedHashSet<String> list = new LinkedHashSet<>(
        Files.readAllLines(input, StandardCharsets.UTF_8));
    System.out.println("Total amount of lines = " + list.size());
    TurkishMorphology morphology = TurkishMorphology.create(
        RootLexicon.builder().addTextDictionaryResources(
            "tr/master-dictionary.dict",
            "tr/non-tdk.dict",
            "tr/proper.dict",
            "tr/proper-from-corpus.dict",
            "tr/abbreviations.dict"
        ).build());
    List<String> toRemove = new ArrayList<>();
    for (DictionaryItem item : morphology.getLexicon()) {
      if (list.contains(item.lemma)) {
        toRemove.add(item.lemma);
      }
    }
    System.out.println("Total amount to remove = " + toRemove.size());
    list.removeAll(toRemove);
    try (PrintWriter pw = new PrintWriter(out.toFile(), "utf-8")) {
      list.forEach(pw::println);
    }
  }

  private void save(Collection<String> data, PrintWriter pw) {
    data.stream().sorted(Turkish.STRING_COMPARATOR_ASC).forEach(pw::println);
  }
}
