code: /* LanguageTool, a natural language style checker
   * Copyright (C) 2006 Daniel Naber (http://www.danielnaber.de)
   * 
   * This library is free software; you can redistribute it and/or
   * modify it under the terms of the GNU Lesser General Public
   * License as published by the Free Software Foundation; either
   * version 2.1 of the License, or (at your option) any later version.
   *
   * This library is distributed in the hope that it will be useful,
   * but WITHOUT ANY WARRANTY; without even the implied warranty of
   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   * Lesser General Public License for more details.
   *
   * You should have received a copy of the GNU Lesser General Public
   * License along with this library; if not, write to the Free Software
   * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
   * USA
   */
  package org.languagetool.tagging;
  
  import morfologik.stemming.Dictionary;
  import morfologik.stemming.WordData;
  import org.jetbrains.annotations.NotNull;
  import org.jetbrains.annotations.Nullable;
  import org.languagetool.*;
  import org.languagetool.rules.spelling.morfologik.MorfologikSpeller;
  import org.languagetool.tools.StringTools;
  
  import java.io.*;
  import java.net.URL;
  import java.util.*;
  
  /**
   * Base tagger using Morfologik binary dictionaries.
   *
   * @author Marcin Milkowski
   */
  public abstract class BaseTagger implements Tagger {
  
    private static final String MANUAL_ADDITIONS_FILE = "/added.txt";
    private static final String CUSTOM_MANUAL_ADDITIONS_FILE = "/added_custom.txt";
    private static final String MANUAL_REMOVALS_FILE = "/removed.txt";
    private static final String CUSTOM_MANUAL_REMOVALS_FILE = "/removed_custom.txt";
  
    protected final WordTagger wordTagger;
    protected final Locale locale;
  
    private final boolean tagLowercaseWithUppercase;
    private final String dictionaryPath;
    private final Dictionary dictionary;
  
    /**
     * Get the filenames for manual additions, e.g., {@code /en/added.txt}.
     * @since 5.0
     */
    @NotNull
    public List<String> getManualAdditionsFileNames() {
      return Arrays.asList(locale.getLanguage() + MANUAL_ADDITIONS_FILE, locale.getLanguage() + CUSTOM_MANUAL_ADDITIONS_FILE);
    }
  
    /**
     * Get the filenames for manual removals, e.g., {@code /en/removed.txt}.
     * @since 5.0
     */
    @NotNull
    public List<String> getManualRemovalsFileNames() {
      return Arrays.asList(locale.getLanguage() + MANUAL_REMOVALS_FILE, locale.getLanguage() + CUSTOM_MANUAL_REMOVALS_FILE);
    }
  
    /** @since 2.9 */
    public BaseTagger(String filename, Locale locale) {
      this(filename, locale, true);
    }
  
    /** @since 2.9 */
    public BaseTagger(String filename, Locale locale, boolean tagLowercaseWithUppercase) {
      this(filename, locale, tagLowercaseWithUppercase, false);
    }
  
    /**
     * @param internTags true if string tags should be interned
     * @since 4.9
     */
    public BaseTagger(String filename, Locale locale, boolean tagLowercaseWithUppercase, boolean internTags) {
      this.dictionaryPath = filename;
      this.locale = locale;
      this.tagLowercaseWithUppercase = tagLowercaseWithUppercase;
      this.dictionary = MorfologikSpeller.getDictionaryWithCaching(filename);
      this.wordTagger = initWordTagger(internTags);
    }
  
    /**
     * @since 2.9
     */
    public String getDictionaryPath() {
      return dictionaryPath;
    }
  
    /**
     * If true, tags from the binary dictionary (*.dict) will be overwritten by manual tags
     * from the plain text dictionary.
     * @since 2.9
     */
    public boolean overwriteWithManualTagger() {
      return false;
    }
  
    protected WordTagger getWordTagger() {
      return wordTagger;
    }
  
    private WordTagger initWordTagger(boolean internTags) {
      MorfologikTagger morfologikTagger = new MorfologikTagger(dictionary, internTags);
      try {
        ManualTagger removalTagger = null;
        InputStream stream = null;
        try {
          for (String file : getManualRemovalsFileNames()) {
            for (URL url : JLanguageTool.getDataBroker().getFromResourceDirAsUrls(file)) {
              if (stream == null) {
                stream = url.openStream();
              } else {
                stream = new SequenceInputStream(stream, url.openStream());
              }
            }
          }
          if (stream != null) {
            removalTagger = new ManualTagger(stream, internTags);
          }
        } finally {
          if (stream != null) {
            stream.close();
          }
        }
  
        stream = null;
        try {
          //long startTime = System.currentTimeMillis();
          for (String file : getManualAdditionsFileNames()) {
            for (URL url : JLanguageTool.getDataBroker().getFromResourceDirAsUrls(file)) {
              if (stream == null) {
                stream = url.openStream();
              } else {
                stream = new SequenceInputStream(stream, url.openStream());
              }
            }
          }
          if (stream != null) {
            ManualTagger manualTagger = new ManualTagger(stream, internTags);
            //long loadTime = System.currentTimeMillis() - startTime;
            //System.out.println("Loading " + getManualAdditionsFileNames() + " took " + loadTime + "ms");
            return new CombiningTagger(morfologikTagger, manualTagger, removalTagger, overwriteWithManualTagger());
          } else {
            return morfologikTagger;
          }
        } finally {
          if (stream != null) {
            stream.close();
          }
        }
      } catch (IOException e) {
        throw new RuntimeException("Could not load manual tagger data from " + getManualAdditionsFileNames(), e);
      }
    }
  
    protected Dictionary getDictionary() {
      return dictionary;
    }
  
    @Override
    public List<AnalyzedTokenReadings> tag(List<String> sentenceTokens)
        throws IOException {
      List<AnalyzedTokenReadings> tokenReadings = new ArrayList<>();
      int pos = 0;
      for (String word : sentenceTokens) {
        List<AnalyzedToken> l = getAnalyzedTokens(word);
        tokenReadings.add(new AnalyzedTokenReadings(l, pos));
        pos += word.length();
      }
      return tokenReadings;
    }
  
    protected List<AnalyzedToken> getAnalyzedTokens(String word) {
      List<AnalyzedToken> result = new ArrayList<>();
      String lowerWord = word.toLowerCase(locale);
      boolean isLowercase = word.equals(lowerWord);
      boolean isMixedCase = StringTools.isMixedCase(word);
      List<AnalyzedToken> taggerTokens = asAnalyzedTokenListForTaggedWords(word, getWordTagger().tag(word));
      List<AnalyzedToken> lowerTaggerTokens =
          ! isLowercase
              ? asAnalyzedTokenListForTaggedWords(word, getWordTagger().tag(lowerWord))
              : taggerTokens;
  
      //normal case:
      addTokens(taggerTokens, result);
      //tag non-lowercase (alluppercase or startuppercase), but not mixedcase word with lowercase word tags:
      if (!isLowercase && !isMixedCase) {
        addTokens(lowerTaggerTokens, result);
      }
      //tag lowercase word with startuppercase word tags:
      if (tagLowercaseWithUppercase
          && lowerTaggerTokens.isEmpty()
          && taggerTokens.isEmpty()
          && isLowercase) {
        List<AnalyzedToken> upperTaggerTokens = asAnalyzedTokenListForTaggedWords(word,
              getWordTagger().tag(StringTools.uppercaseFirstChar(word)));
        if (!upperTaggerTokens.isEmpty()) {
          addTokens(upperTaggerTokens, result);
        }
      }
      // Additional language-dependent-tagging:
      if (result.isEmpty()) {
        List<AnalyzedToken> additionalTaggedTokens = additionalTags(word, getWordTagger());
        addTokens(additionalTaggedTokens, result);
      }
      if (result.isEmpty()) {
        result.add(new AnalyzedToken(word, null, null));
      }
      return result;
    }
  
    protected List<AnalyzedToken> asAnalyzedTokenList(String word, List<WordData> wdList) {
      List<AnalyzedToken> aTokenList = new ArrayList<>();
      for (WordData wd : wdList) {
        aTokenList.add(asAnalyzedToken(word, wd));
      }
      return aTokenList;
    }
  
    protected List<AnalyzedToken> asAnalyzedTokenListForTaggedWords(String word, List<TaggedWord> taggedWords) {
      List<AnalyzedToken> aTokenList = new ArrayList<>();
      for (TaggedWord taggedWord : taggedWords) {
        aTokenList.add(asAnalyzedToken(word, taggedWord));
      }
      return aTokenList;
    }
  
    protected AnalyzedToken asAnalyzedToken(String word, WordData wd) {
      String tag = StringTools.asString(wd.getTag());
      // Remove frequency data from tags (if exists)
      // The frequency data is in the last byte (without a separator)
      if (dictionary.metadata.isFrequencyIncluded() && tag.length() > 1) {
        tag = tag.substring(0, tag.length() - 1);
      }
      return new AnalyzedToken( word, tag, StringTools.asString(wd.getStem()));
    }
  
    private AnalyzedToken asAnalyzedToken(String word, TaggedWord taggedWord) {
      return new AnalyzedToken(word, taggedWord.getPosTag(), taggedWord.getLemma());
    }
  
    //please do not make protected, this breaks other languages
    private void addTokens(List<AnalyzedToken> taggedTokens, List<AnalyzedToken> l) {
      if (taggedTokens != null) {
        for (AnalyzedToken at : taggedTokens) {
          l.add(at);
        }
      }
    }
  
    @Override
    public final AnalyzedTokenReadings createNullToken(String token, int startPos) {
      return new AnalyzedTokenReadings(new AnalyzedToken(token, null, null), startPos);
    }
  
    @Override
    public AnalyzedToken createToken(String token, String posTag) {
      return new AnalyzedToken(token, posTag, null);
    }
  
    /**
     * Allows additional tagging in some language-dependent circumstances
     * @param word The word to tag
     * @return Returns list of analyzed tokens with additional tags, or {@code null}
     */
    @Nullable
    protected List<AnalyzedToken> additionalTags(String word, WordTagger wordTagger) {
      return null;
    }
  
  }
  