yago3-master/src/main/java/utils/demonyms/DemonymsTableExtractor.java
package utils.demonyms;

import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import basics.FactComponent;

public class DemonymsTableExtractor {

  public static void main(String[] args) {

    System.out.println("# generated by DenonymsTableExtractor");
    LinkedHashMap<String, String> patterns = getLocationPatterns();

    for (Entry<String, String> location : patterns.entrySet()) {
      StringBuilder sb = new StringBuilder();
      sb.append("\"");
      sb.append(location.getValue());
      sb.append("\" <_categoryPattern> \"$0 <isCitizenOf> ");
      sb.append(location.getKey());
      sb.append("\" .");
      System.out.println(sb.toString());
    }
  }

  public static LinkedHashMap<String, String> getLocationPatterns() {
    Map<String, Set<String>> countryToAdjective = null;
    LinkedHashMap<String, String> result = new LinkedHashMap<>();

    //content += Wiki.getPage("List_of_adjectival_and_demonymic_forms_of_place_names", "en");
    //content += Wiki.getPage("List_of_adjectivals_and_demonyms_for_cities", "en");
    // Wiki.getPage("List_of_demonyms_for_U.S._states", "en");
    String content = DemonymPattern.getPage("List_of_adjectival_and_demonymic_forms_for_countries_and_nations", "en");

    Set<String> ignore = new HashSet<>(Arrays.asList("British", "Irish", "Chinese", "Dominican", "Congolese", "Channel Island"));
    countryToAdjective = extract(content, ignore);
    fix(countryToAdjective);
    checkForDuplicates(countryToAdjective);
    result.putAll(categoryPatterns(countryToAdjective));

    content = DemonymPattern.getPage("List_of_adjectivals_and_demonyms_for_former_regions", "en");

    System.out.println("# former regions");
    countryToAdjective.forEach((k, v) -> ignore.addAll(v));

    countryToAdjective = extract(content, ignore);
    result.putAll(categoryPatterns(countryToAdjective));

    return result;
  }

  private static Map<String, Set<String>> extract(String content, Set<String> ignore) {
    List<WikiTable> tables = WikiTable.getTables(content);

    WikiTable t0 = tables.get(0);

    int subjectCol = 0;
    int adjCol = 1;
    int demonymCol = 2;

    Map<String, Set<String>> countryToAdjective = new HashMap<>();
    for (List<String> row : t0.rows) {

      Matcher m = DemonymPattern.wikiLink.matcher(row.get(subjectCol));
      if (m.find()) {
        String subject = m.group("page").replaceAll("#.*", "");

        DemonymsPageExtractor.Helper h = new DemonymsPageExtractor.Helper();
        HashSet<String> adjectives = new HashSet<>();
        h.remaining = row.get(adjCol);
        extract(h);
        adjectives.addAll(h.found);

        /*h.remaining = row.get(demonymCol);
        extract(h);
        demonyms.addAll(h.found);*/

        adjectives.add(subject);

        adjectives.removeAll(ignore);
        if (adjectives.size() > 0) {
          countryToAdjective.put(FactComponent.forWikipediaTitle(subject), adjectives);
        }
      }
    }

    return countryToAdjective;
  }

  private static void checkForDuplicates(Map<String, Set<String>> countryToAdjective) {
    Set<String> seen = new HashSet<>(), duplicates = new HashSet<>();
    for (Set<String> vals : countryToAdjective.values()) {
      Set<String> copy = new HashSet<>(vals);
      copy.retainAll(seen);
      duplicates.addAll(copy);
      seen.addAll(vals);
    }

    System.out.println("# duplicate adjectives: " + duplicates);
  }

  private static void fix(Map<String, Set<String>> countryToAdjective) {
    countryToAdjective.remove("<Lithuanian>");
    countryToAdjective.remove("<Surinam>");

    add(countryToAdjective, "<British_Antarctic_Territory>", "British Antarctic Territory");
    add(countryToAdjective, "<French_Guiana>", "French Guianan");
    add(countryToAdjective, "<Gibraltar>", "Gibraltarian");
    add(countryToAdjective, "<Guadeloupe>", "Guadeloupean");
    add(countryToAdjective, "<Guinea-Bissau>", "Bissau-Guinean");
    add(countryToAdjective, "<Luxembourg>", "Luxembourgian");
    add(countryToAdjective, "<Lithuania>", "Lithuanian");
    add(countryToAdjective, "<Monaco>", "Monegasque");
    //add(countryToAdjective, "<Myanmar>", "Burmese");
    add(countryToAdjective, "<Northern_Cyprus>", "Turkish Cypriot");
    add(countryToAdjective, "<Somalia>", "Somalian");
    add(countryToAdjective, "<Suriname>", "Surinamese");
    add(countryToAdjective, "<Taiwan>", "Taiwanese");
    // ambiguous
    add(countryToAdjective, "<China>", "Chinese");
    add(countryToAdjective, "<Ireland>", "Irish");
    add(countryToAdjective, "<United_Kingdom>", "British");

  }

  /**
   * Generates the category patterns
   * @param countryToAdjective map from country to adjective(s), e.g. France -> French
   * @return map from country to regex, e.g. (simplified) France -> French .*|.* of France
   */
  private static Map<String, String> categoryPatterns(Map<String, Set<String>> countryToAdjective) {
    Map<String, String> result = new TreeMap<>();

    for (String key : countryToAdjective.keySet()) {
      StringBuilder sb = new StringBuilder();
      sb.append("\\\\b(");
      int idx = 0;
      for (String val : countryToAdjective.get(key)) {
        if (idx++ > 0) {
          sb.append("|");
        }
        sb.append(val);
      }
      sb.append(")\\\\b .*");
      sb.append("|.* (from|of)( the)? ");
      sb.append(FactComponent.stripBracketsAndLanguage(key).replaceFirst("[#].*", "").replaceFirst("^The ", ""));
      result.put(key, sb.toString());
    }

    return result;
  }

  private static void add(Map<String, Set<String>> countryToAdjective, String subj, String adj) {
    Set<String> adjs = countryToAdjective.computeIfAbsent(subj, k -> new HashSet<>());
    adjs.add(adj);
    adjs.add(FactComponent.stripBracketsAndLanguage(subj));
  }

  static Pattern wikiLinkWithSuffix = Pattern.compile(DemonymPattern.wikiLink.pattern() + "(?<suffix>[a-zA-z]*)");

  static Pattern text = Pattern.compile("[A-z][a-z]{3,30}");

  private static void extract(DemonymsPageExtractor.Helper helper) {
    helper.remaining = DemonymPattern.refPattern.matcher(helper.remaining).replaceAll("");
    helper.remaining = DemonymPattern.bracesPattern.matcher(helper.remaining).replaceAll("");
    helper.remaining = helper.remaining.replaceAll("}}", "");

    extractFromLinks(helper);
  }

  private static void extractFromLinks(DemonymsPageExtractor.Helper helper) {
    Matcher m = wikiLinkWithSuffix.matcher(helper.remaining);
    while (m.find()) {
      String name = m.group("anchor");
      if (name == null) name = m.group("page");
      String suffix = m.group("suffix");

      helper.found.add(name + suffix);
    }
    helper.remaining = m.replaceAll("");

    if (helper.found.isEmpty()) {
      helper.remaining = helper.remaining.replaceAll("\\(.*\\)", "");
      m = text.matcher(helper.remaining);
      while (m.find()) {
        helper.found.add(m.group());
      }
    }
  }

}
