code: /*
   * Carnegie Mellon University
   * Copyright (c) 2004, 2010
   * 
   * This software is distributed under the terms of the GNU Lesser General
   * Public License.  See the included COPYING and COPYING.LESSER files.
   * 
   */
  
  package edu.cmu.meteor.util;
  
  import java.io.BufferedReader;
  import java.io.IOException;
  import java.io.InputStreamReader;
  import java.net.URL;
  import java.util.Hashtable;
  import java.util.StringTokenizer;
  import java.util.regex.Pattern;
  
  /**
   * This normalizer is based on the Moses tokenizer Rev 3612 with additions to
   * improve MT evaluation accuracy.
   * 
   * Nonbreaking prefix lists taken from Moses Rev 3881.
   * 
   * Additions: quote and dash normalization, lossy stripping of dashes and dots
   * in words. Departures from Moses tokenizer marked with "New".
   */
  public class Normalizer {
  
  	// Tokenization patterns
  
  	private static String s_space = " ";
  
  	// Lists of alpha and alphanumeric characters, including Unicode ranges
      //                            Western European         Cyrillic
  	private static String alpha = "A-Za-zŠŽšžŸÀ-ÖØ-öø-ž" + "Ѐ-ӿԀ-ԧꙀ-ꙮ꙾-ꚗᴀ-ᵿ";
  	private static String alnum = "0-9A-Za-zŠŽšžŸÀ-ÖØ-öø-ž" + "Ѐ-ӿԀ-ԧꙀ-ꙮ꙾-ꚗᴀ-ᵿ";
  
  	// List of special case characters not to wrap in whitespace
  	private static Pattern r_sep_other = Pattern.compile("([^" + alnum
  			+ "\\s\\.\\'\\`\\,\\-\\‘\\’])");
  	private static String s_sep_other = " $1 ";
  
  	private static Pattern r_multi_dot = Pattern.compile("\\.([\\.]+)");
  	private static String s_multi_dot = " DOTMULTI$1";
  	private static String s_multi_dot2 = "DOTMULTI.";
  	private static Pattern r_multi_dot2 = Pattern
  			.compile("DOTMULTI\\.([^\\.])");
  	private static String s_multi_dot3 = "DOTDOTMULTI $1";
  	private static String s_multi_dot4 = "DOTDOTMULTI";
  	private static String s_multi_dot5 = "DOTMULTI";
  	private static String s_multi_dot6 = ".";
  
  	private static Pattern r_comma = Pattern
  			.compile("([^\\p{Digit}])[,]([^\\p{Digit}])");
  	private static String s_comma = "$1 , $2";
  	private static Pattern r_comma2 = Pattern
  			.compile("([\\p{Digit}])[,]([^\\p{Digit}])");
  	private static Pattern r_comma3 = Pattern
  			.compile("([^\\p{Digit}])[,]([\\p{Digit}])");
  
  	// Single quotes to normalize
  	private static Pattern r_quote_norm = Pattern.compile("([`‘’])");
  	private static String s_quote_norm = "'";
  	// Double quotes to normalize
  	private static Pattern r_quote_norm2 = Pattern.compile("([“”]|'')");
  	private static String s_quote_norm2 = " \" ";
  
  	// Dashes to normalize
  	private static String s_dash_norm = "–";
  	private static String s_dash_norm2 = "-";
  	private static String s_dash_norm3 = "--";
  
  	private static Pattern r_cont_en = Pattern.compile("([^" + alpha
  			+ "])[']([^" + alpha + "])");
  	private static String s_cont_en = "$1 ' $2";
  	private static Pattern r_cont_en2 = Pattern.compile("([^" + alpha
  			+ "\\p{Digit}])[']([" + alpha + "])");
  	private static Pattern r_cont_en3 = Pattern.compile("([" + alpha
  			+ "])[']([^" + alpha + "])");
  	private static Pattern r_cont_en4 = Pattern.compile("([" + alpha
  			+ "])[']([" + alpha + "])");
  	private static String s_cont_en2 = "$1 '$2";
  	// 1990's etc.
  	private static Pattern r_cont_en5 = Pattern
  			.compile("([\\p{Digit}])[']([s])");
  
  	private static Pattern r_cont_fr = Pattern.compile("([^" + alpha
  			+ "])[']([^" + alpha + "])");
  	private static String s_cont_fr = "$1 ' $2";
  	private static Pattern r_cont_fr2 = Pattern.compile("([^" + alpha
  			+ "])[']([" + alpha + "])");
  	private static Pattern r_cont_fr3 = Pattern.compile("([" + alpha
  			+ "])[']([^" + alpha + "])");
  	private static Pattern r_cont_fr4 = Pattern.compile("([" + alpha
  			+ "])[']([" + alpha + "])");
  	private static String s_cont_fr2 = "$1' $2";
  
  	private static String s_cont_other1 = "'";
  	private static String s_cont_other2 = " ' ";
  
  	private static Pattern r_punct_strip = Pattern.compile("[^" + alnum + "]");
  	private static String s_punct_strip = " ";
  
  	private static Pattern r_rm_dash = Pattern.compile("([" + alnum
  			+ "\\.])[\\-]([" + alnum + "])");
  	private static String s_rm_dash = "$1 $2";
  
      //                                                  Unicode spaces
  	private static Pattern r_white = Pattern.compile("[               　 ]+");
  	private static String s_white = " ";
  
  	// Nonbreaking prefixes
  	private static Hashtable<String, Integer> nbpDict = null;
  	private static int nbpLangID = Constants.LANG_OTHER;
  	private static String s_nbp = ".";
  	private static String s_nbp2 = "";
  	private static Pattern r_nbp1 = Pattern.compile("[" + alpha + "]");
  	private static Pattern r_nbp2 = Pattern.compile("^[\\p{Lower}]");
  	private static Pattern r_nbp3 = Pattern.compile("^[0-9]+");
  	private static String s_nbp3 = " .";
  	private static String s_nbp4 = " ";
  
  	// Should refactor if intending to normalize lines of alternating languages.
  	// This implementation works for the MT Evaluation use case.
  	private static Hashtable<String, Integer> nbpList(int langID) {
  
  		if (nbpDict != null && nbpLangID == langID)
  			return nbpDict;
  
  		// New nbp list
  		nbpDict = new Hashtable<String, Integer>();
  		nbpLangID = Constants.LANG_OTHER;
  
  		// Try to load the list
  		try {
  			URL nbpFileURL = new URL(Constants.DEFAULT_NBP_DIR_URL.toString()
  					+ "/" + Constants.getLanguageName(langID) + ".prefixes");
  			BufferedReader in = new BufferedReader(new InputStreamReader(
  					nbpFileURL.openStream(), "UTF-8"));
  			String line;
  			while ((line = in.readLine()) != null) {
  				StringTokenizer tok = new StringTokenizer(line);
  				// Skip empty entries
  				if (tok.countTokens() == 0)
  					continue;
  				String pre = tok.nextToken();
  				// Skip comments
  				if (pre.startsWith("#"))
  					continue;
  				// Check type
  				int type = Constants.NBP_ANY;
  				if (tok.hasMoreTokens()
  						&& tok.nextToken().equals("#NUMERIC_ONLY#"))
  					type = Constants.NBP_NUM_ONLY;
  				nbpDict.put(pre, type);
  			}
  			in.close();
  
  			// Set language for loaded nbp list
  			nbpLangID = langID;
  		} catch (Exception ex) {
  			System.err
  					.println("Error: Nonbreaking prefix list could not be loaded:");
  			ex.printStackTrace();
  		}
  
  		return nbpDict;
  	}
  
  	// Normalization for western languages. Checks if language is non-western
  	// (and supported) and handles appropriately
  	public static String normalizeLine(String line, int langID,
  			boolean keepPunctuation) {
  
  		if (!Constants.isSupported(langID)) {
  			System.err
  					.println("Error: Pre-process the input files and run Meteor without the -norm option.");
  			String lang = "";
  			try {
  				lang = Constants.getLanguageName(langID);
  			} catch (Exception ex) {
  				// Unknown language, leave blank
  			}
  			throw new RuntimeException("No normalizer for language (" + lang
  					+ ")");
  		}
  
  		// Special handling of non-western languages
  		if (langID == Constants.LANG_AR_BW_RED || langID == Constants.LANG_OTHER)
  			return normalizeNonWestern(line, keepPunctuation);
  
  		// Wrap line in whitespace
  		String workingLine = s_space + line + s_space;
  
  		// Separate all non-alphanumeric characters that aren't special cases
  		workingLine = r_sep_other.matcher(workingLine).replaceAll(s_sep_other);
  
  		// Escape multiple dots
  		workingLine = r_multi_dot.matcher(workingLine).replaceAll(s_multi_dot);
  		while (workingLine.contains(s_multi_dot2)) {
  			workingLine = r_multi_dot2.matcher(workingLine).replaceAll(
  					s_multi_dot3);
  			workingLine = workingLine.replace(s_multi_dot2, s_multi_dot4);
  		}
  
  		// Separate commas except within numbers
  		workingLine = r_comma.matcher(workingLine).replaceAll(s_comma);
  		workingLine = r_comma2.matcher(workingLine).replaceAll(s_comma);
  		workingLine = r_comma3.matcher(workingLine).replaceAll(s_comma);
  
  		// New: Normalize quotes
  		workingLine = r_quote_norm.matcher(workingLine)
  				.replaceAll(s_quote_norm);
  		workingLine = r_quote_norm2.matcher(workingLine).replaceAll(
  				s_quote_norm2);
  
  		// New: Normalize dashes
  		workingLine = workingLine.replace(s_dash_norm, s_dash_norm2);
  		workingLine = workingLine.replace(s_dash_norm3, s_dash_norm2);
  
  		// New: Remove dashes between words so systems can get partial credit
  		// Find dashes alphanum-alphanum or dot-alphanum (U.S.-based)
  		workingLine = r_rm_dash.matcher(workingLine).replaceAll(s_rm_dash);
  
  		// Handle apostrophes
  		if (langID == Constants.LANG_EN) {
  			// English splits contractions right
  			workingLine = r_cont_en.matcher(workingLine).replaceAll(s_cont_en);
  			workingLine = r_cont_en2.matcher(workingLine).replaceAll(s_cont_en);
  			workingLine = r_cont_en3.matcher(workingLine).replaceAll(s_cont_en);
  			workingLine = r_cont_en4.matcher(workingLine)
  					.replaceAll(s_cont_en2);
  			workingLine = r_cont_en5.matcher(workingLine)
  					.replaceAll(s_cont_en2);
  		} else if (langID == Constants.LANG_FR) {
  			// French splits contractions left
  			workingLine = r_cont_fr.matcher(workingLine).replaceAll(s_cont_fr);
  			workingLine = r_cont_fr2.matcher(workingLine).replaceAll(s_cont_fr);
  			workingLine = r_cont_fr3.matcher(workingLine).replaceAll(s_cont_fr);
  			workingLine = r_cont_fr4.matcher(workingLine)
  					.replaceAll(s_cont_fr2);
  		} else {
  			// All others split both
  			workingLine = workingLine.replace(s_cont_other1, s_cont_other2);
  		}
  
  		// Split words, dots still attached
  		StringTokenizer tok = new StringTokenizer(workingLine);
  		String[] words = new String[tok.countTokens()];
  		for (int i = 0; i < words.length; words[i++] = tok.nextToken())
  			;
  		StringBuilder sb = new StringBuilder();
  
  		// Rebuild line, breaking dots unless cased as nonbreaking
  		Hashtable<String, Integer> nbp = nbpList(langID);
  		for (int i = 0; i < words.length; i++) {
  			if (words[i].length() > 1 && words[i].endsWith(s_nbp)) {
  				String pre = words[i].substring(0, words[i].length() - 1);
  				Integer type = nbp.get(pre);
  				// Mixed (U.S.)
  				if (pre.contains(s_nbp) && r_nbp1.matcher(pre).find()) {
  					// New: Drop dots to match undotted output (U.S. -> US)
  					sb.append(words[i].replace(s_nbp, s_nbp2));
  				}// Always nonbreaking or followed by lowercase (trick to catch
  					// unlisted items such as Inc.)
  				else if ((type != null && type == Constants.NBP_ANY)
  						|| (i < words.length - 1 && r_nbp2
  								.matcher(words[i + 1]).find())) {
  					// No change
  					sb.append(words[i]);
  				} // Nonbreaking when followed by number
  				else if ((type != null && type == Constants.NBP_NUM_ONLY)
  						&& (i < words.length - 1 && (r_nbp3
  								.matcher(words[i + 1]).find()))) {
  					// No change
  					sb.append(words[i]);
  				} else {
  					// Split dot
  					sb.append(pre);
  					sb.append(s_nbp3);
  				}
  			} else {
  				// No nbp
  				sb.append(words[i]);
  			}
  			sb.append(s_nbp4);
  		}
  		workingLine = sb.toString();
  
  		// Unescape multiple dots
  		while (workingLine.contains(s_multi_dot4)) {
  			workingLine = workingLine.replace(s_multi_dot4, s_multi_dot2);
  		}
  		workingLine = workingLine.replace(s_multi_dot5, s_multi_dot6);
  
  		// New: Remove punctuation if requested
  		if (!keepPunctuation) {
  			workingLine = r_punct_strip.matcher(workingLine).replaceAll(
  					s_punct_strip);
  		}
  
  		// Normalize whitespace
  		workingLine = r_white.matcher(workingLine).replaceAll(s_white).trim();
  		return workingLine;
  	}
  
  	// Symbol ranges excluding comma and dot
  	private static Pattern r_punct_nonwest = Pattern
  			.compile("([\\!-\\+\\-\\/\\:-\\@\\[-\\`\\{-¿\u060C])");
  	private static String s_punct_nonwest = " $1 ";
  	private static String s_punct_nonwest2 = " ";
  	private static String s_punct_nonwest3 = ".";
  
  	private static Pattern r_dot_nonwest = Pattern
  			.compile("([^\\p{Digit}])[\\.]([^\\p{Digit}])");
  	private static String s_dot_nonwest = "$1 . $2";
  	private static Pattern r_dot_nonwest2 = Pattern
  			.compile("([\\p{Digit}])[\\.]([^\\p{Digit}])");
  	private static Pattern r_dot_nonwest3 = Pattern
  			.compile("([^\\p{Digit}])[\\.]([\\p{Digit}])");
  
  	// For non-western languages, only tokenize western punctuation
  	private static String normalizeNonWestern(String line,
  			Boolean keepPunctuation) {
  
  		// Wrap line in whitespace
  		String workingLine = s_space + line + s_space;
  
  		// Escape multiple dots
  		workingLine = r_multi_dot.matcher(workingLine).replaceAll(s_multi_dot);
  		while (workingLine.contains(s_multi_dot2)) {
  			workingLine = r_multi_dot2.matcher(workingLine).replaceAll(
  					s_multi_dot3);
  			workingLine = workingLine.replace(s_multi_dot2, s_multi_dot4);
  		}
  
  		// Separate commas except within numbers
  		workingLine = r_comma.matcher(workingLine).replaceAll(s_comma);
  		workingLine = r_comma2.matcher(workingLine).replaceAll(s_comma);
  		workingLine = r_comma3.matcher(workingLine).replaceAll(s_comma);
  
  		// Normalize quotes
  		workingLine = r_quote_norm.matcher(workingLine)
  				.replaceAll(s_quote_norm);
  		workingLine = r_quote_norm2.matcher(workingLine).replaceAll(
  				s_quote_norm2);
  
  		// Normalize dashes
  		workingLine = workingLine.replace(s_dash_norm, s_dash_norm2);
  		workingLine = workingLine.replace(s_dash_norm3, s_dash_norm2);
  
  		// Separate dots except within numbers
  		workingLine = r_dot_nonwest.matcher(workingLine).replaceAll(
  				s_dot_nonwest);
  		workingLine = r_dot_nonwest2.matcher(workingLine).replaceAll(
  				s_dot_nonwest);
  		workingLine = r_dot_nonwest3.matcher(workingLine).replaceAll(
  				s_dot_nonwest);
  
  		// Tokenize all other punctuation (last to avoid confusion)
  		workingLine = r_punct_nonwest.matcher(workingLine).replaceAll(
  				s_punct_nonwest);
  
  		// Remove punctuation if requested
  		if (!keepPunctuation) {
  			workingLine = r_punct_nonwest.matcher(workingLine).replaceAll(
  					s_punct_nonwest2);
  			workingLine = workingLine.replace(s_punct_nonwest3,
  					s_punct_nonwest2);
  		}
  
  		// Unescape multiple dots
  		while (workingLine.contains(s_multi_dot4)) {
  			workingLine = workingLine.replace(s_multi_dot4, s_multi_dot2);
  		}
  		workingLine = workingLine.replace(s_multi_dot5, s_multi_dot6);
  
  		// Normalize whitespace
  		workingLine = r_white.matcher(workingLine).replaceAll(s_white);
  		workingLine = workingLine.trim();
  
  		return workingLine;
  	}
  
  	// SGML Patterns
  
  	private static Pattern r_quot = Pattern.compile("&quot;",
  			Pattern.CASE_INSENSITIVE);
  	private static Pattern r_apos = Pattern.compile("&apos;",
  			Pattern.CASE_INSENSITIVE);
  	private static Pattern r_lt = Pattern.compile("&lt;",
  			Pattern.CASE_INSENSITIVE);
  	private static Pattern r_gt = Pattern.compile("&gt;",
  			Pattern.CASE_INSENSITIVE);
  	private static Pattern r_amp = Pattern.compile("&amp;",
  			Pattern.CASE_INSENSITIVE);
  
  	private static String quot = "\"";
  	private static String apos = "'";
  	private static String lt = "<";
  	private static String gt = ">";
  	private static String amp = "&";
  
  	// Call separately for SGML input only
  	public static String unescapeSGML(String line) {
  		String workingLine = line;
  		workingLine = r_quot.matcher(workingLine).replaceAll(quot);
  		workingLine = r_apos.matcher(workingLine).replaceAll(apos);
  		workingLine = r_lt.matcher(workingLine).replaceAll(lt);
  		workingLine = r_gt.matcher(workingLine).replaceAll(gt);
  		workingLine = r_amp.matcher(workingLine).replaceAll(amp);
  		return workingLine;
  	}
  
  	// Test the normalizer
  	public static void main(String[] args) throws IOException {
  
  		if (args.length < 2) {
  			System.out.println("Usage: Normalizer lang punct");
  			System.out.println("where puct is true/false");
  			return;
  		}
  
  		int langID = Constants.getLanguageID(Constants
  				.normLanguageName(args[0]));
  		boolean punct = Boolean.parseBoolean(args[1]);
  		BufferedReader in = new BufferedReader(new InputStreamReader(System.in));
  		String line;
  		while ((line = in.readLine()) != null)
  			System.out.println(Normalizer.normalizeLine(line, langID, punct));
  	}
  }
  