yago3-master/src/main/java/utils/demonyms/DemonymsPageExtractor.java
package utils.demonyms;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.stream.Collectors;

import javatools.administrative.Announce;

/**
 * Class with demonym maps, and code to generate those from Wikipedia.
 *
 * @author Thomas Rebele
 *
 */
public class DemonymsPageExtractor {

  static class Helper {

    String remaining = "";

    List<String> found = new ArrayList<>();
  }

  public static void main(String[] args) throws IOException {
    Map<String, List<String>> subjectToDemonym = new HashMap<>();

    System.out.println("/* this was generated by DemonymsPageExtractor */");
    System.out.println("map = subjectToDemonym;");
    // output as java code
    for (String key : subjectToDemonym.keySet()) {
      StringBuilder sb = new StringBuilder();
      for (String val : subjectToDemonym.get(key)) {
        if (sb.length() > 0) {
          sb.append(", ");
        }
        sb.append("\"" + val + "\"");
      }
      System.out.println("Utility.merge(map, \"" + key + "\", Arrays.asList(" + sb.toString() + "));");
    }
  }

  public static Map<String, List<String>> getLocationToDemonym() {
    String content = DemonymPattern.getPage("Demonym", "en");

    // extracting country and demonyms
    List<String> lines = Arrays.asList(content.split("\\R+"));
    lines = lines.stream().filter(str -> str.contains("→")).collect(Collectors.toList());

    Map<String, List<String>> subjectToDemonym = new HashMap<>();
    for (String line : lines) {
      String[] parts = line.split("→");

      // extract conutry
      String country = parts[0];
      Matcher countryMatcher = DemonymPattern.wikiLink.matcher(country);
      if (countryMatcher.find()) {
        country = countryMatcher.group("page");
      }

      if (parts.length > 2 && !"Finland".equals(country)) {
        Announce.warning("didn't expect two → characters in ", country, ": ", line);
      }

      // extract demonyms
      List<String> demonyms = new ArrayList<>();
      Helper t = new Helper();
      t.remaining = parts[1];
      extract(t);
      t.found.remove("or");
      subjectToDemonym.put(country, t.found);

      // print not completely processed entries
      if (t.remaining.length() > 0) {
        Announce.debug(country + ": " + demonyms + " \t\t\n" + line + "\n" + t.remaining + "\n");
      }
    }
    return subjectToDemonym;
  }

  static void extract(Helper helper) {
    helper.remaining = helper.remaining.trim();
    if (helper.remaining.length() == 0) return;

    helper.remaining = DemonymPattern.refPattern.matcher(helper.remaining).replaceAll("");
    helper.remaining = DemonymPattern.bracesPattern.matcher(helper.remaining).replaceAll("");
    helper.remaining = helper.remaining.replaceAll("}}", "");
    helper.remaining = helper.remaining.replaceAll("\\[North \\/ South\\]", "");

    extractSimple(helper);
    extractFromParanthesis(helper);
    extractFromLink(helper);
    extractFromQuote(helper);

    helper.remaining = helper.remaining.replaceAll("''\\)", "");

    extractSimple(helper);

    List<String> parts = Arrays.asList(helper.remaining.split("[,\\/]"));
    if (parts.size() > 1) {
      for (String str : parts) {
        helper.remaining = str;
        extract(helper);
      }
    }
    helper.found.remove("");
  }

  private static void extractFromLink(Helper helper) {
    Matcher m = DemonymPattern.wikiLink.matcher(helper.remaining);
    while (m.find()) {
      helper.found.add(m.group("anchor"));
    }
    helper.remaining = m.replaceAll("");
  }

  private static void extractSimple(Helper helper) {
    helper.remaining = helper.remaining.trim();
    if (DemonymPattern.simplePattern.matcher(helper.remaining).matches()) {
      helper.found.add(helper.remaining);
      helper.remaining = "";
    }
  }

  private static void extractFromQuote(Helper helper) {
    Matcher mq = DemonymPattern.quotePattern.matcher(helper.remaining);
    while (mq.find()) {
      helper.found.add(mq.group(1));
    }
    helper.remaining = mq.replaceAll("");
  }

  private static void extractFromParanthesis(Helper helper) {
    Matcher mp = DemonymPattern.paranthesisPattern.matcher(helper.remaining);
    while (mp.find()) {
      Matcher mq = DemonymPattern.quotePattern.matcher(mp.group());
      while (mq.find()) {
        helper.found.add(mq.group(1));
      }
    }
    helper.remaining = mp.replaceAll("").trim();
  }

}
