code: ///**
  // * 
  // */
  //package edu.berkeley.nlp.PCFGLA;
  //
  //import java.util.Arrays;
  //import java.util.List;
  //
  //import edu.berkeley.nlp.PCFGLA.SimpleLexicon.IntegerIndexer;
  //import edu.berkeley.nlp.PCFGLA.smoothing.Smoother;
  //import edu.berkeley.nlp.syntax.StateSet;
  //import edu.berkeley.nlp.syntax.Tree;
  //import fig.basic.Indexer;
  //
  ///**
  // * @author petrov
  // * each word's tagging probability is the sum of the (word,tag) score and the (signature,tag) score
  // *
  // */
  //public class HierarchicalCombinedLexicon extends HierarchicalLexicon{
  //	private static final long serialVersionUID = 1L;
  //	protected int knownWordCount;
  //	/**
  //	 * @param numSubStates
  //	 * @param threshold
  //	 */
  //	public HierarchicalCombinedLexicon(short[] numSubStates, int knownWordCount) {
  //		super(numSubStates, 0);
  //		this.knownWordCount = knownWordCount;
  //	}
  //	
  //	public HierarchicalCombinedLexicon(short[] numSubStates, int smoothingCutoff, double[] smoothParam, 
  //			Smoother smoother, StateSetTreeList trainTrees, int knownWordCount) {
  //  	this(numSubStates, knownWordCount);
  //  	init(trainTrees);
  //  }
  //
  //
  //  /**
  //	 * @param previousLexicon
  //	 */
  //	public HierarchicalCombinedLexicon(SimpleLexicon previousLexicon, int knownWordCount) {
  //		super(previousLexicon);
  //		this.knownWordCount = knownWordCount;
  //	}
  //	
  //	public HierarchicalCombinedLexicon newInstance() {
  //		return new HierarchicalCombinedLexicon(this.numSubStates,this.knownWordCount);
  //	}
  //
  ////	public double[] score(String word, short tag, int loc, boolean noSmoothing, boolean isSignature) {
  ////		int globalWordIndex = wordIndexer.indexOf(word);
  ////		int globalSigIndex = wordIndexer.indexOf(getSignature(word, loc));
  ////		return score(globalWordIndex, globalSigIndex, tag, loc, noSmoothing, isSignature);
  ////	}
  //	
  //
  //	
  //	public double[] score(StateSet stateSet, short tag, boolean noSmoothing, boolean isSignature) {
  ////		String sig = getSignature(stateSet.getWord(), stateSet.from);
  ////		if (stateSet.sigIndex != wordIndexer.indexOf(sig));
  ////			System.out.println("problem, signatures dont match!");
  //		if (stateSet.wordIndex == -2) {
  //			String word = stateSet.getWord();
  //			stateSet.wordIndex = (short)wordIndexer.indexOf(word);
  //			stateSet.sigIndex = (short)wordIndexer.indexOf(getSignature(word,stateSet.from));
  //		}
  //		return score(stateSet.wordIndex, stateSet.sigIndex, tag, stateSet.from, noSmoothing, isSignature);
  //	}
  //	
  //
  //	public double[] score(int globalWordIndex, int globalSigIndex, short tag, int loc, boolean noSmoothing, boolean isSignature) {
  //		double[] res = new double[numSubStates[tag]];
  //		if (globalWordIndex!=-1) {
  //			int tagSpecificWordIndex = tagWordIndexer[tag].indexOf(globalWordIndex);
  //			if (tagSpecificWordIndex!=-1){
  //				for (int i=0; i<numSubStates[tag]; i++){
  //					res[i] = scores[tag][i][tagSpecificWordIndex]; 
  //				}
  //			}
  //			else {
  //				Arrays.fill(res, 1.0);
  //			}
  //		} else {
  //				Arrays.fill(res, 1.0);
  //		} 
  //		if (globalWordIndex>=0 && wordCounter[globalWordIndex]>knownWordCount) return res;
  //		if (globalSigIndex!=-1) {
  //			int tagSpecificWordIndex = tagWordIndexer[tag].indexOf(globalSigIndex);
  //			if (tagSpecificWordIndex!=-1){
  //				for (int i=0; i<numSubStates[tag]; i++){
  //					res[i] *= scores[tag][i][tagSpecificWordIndex]; 
  //				}
  //			}
  //		} else{
  //			System.out.println("unseen sig");
  //		}
  //		if (smoother!=null) smoother.smooth(tag,res);
  //		return res;
  //  } 
  //
  //	public double[] scoreWord(StateSet stateSet, int tag){
  //		return scoreWord(stateSet.wordIndex, tag);
  //	}
  //	
  //  public double[] scoreWord(String word, int tag) {
  //		int globalWordIndex = wordIndexer.indexOf(word);
  //		return scoreWord(globalWordIndex, tag);
  //  }
  //
  //	public double[] scoreWord(int globalWordIndex, int tag){
  //		double[] res = new double[numSubStates[tag]];
  //		if (globalWordIndex!=-1) {
  //			int tagSpecificWordIndex = tagWordIndexer[tag].indexOf(globalWordIndex);
  //			if (tagSpecificWordIndex!=-1){
  //				for (int i=0; i<numSubStates[tag]; i++){
  //					res[i] = scores[tag][i][tagSpecificWordIndex]; 
  //				}
  //			}
  //		} else {
  //			Arrays.fill(res, 1.0);
  //		}
  //		return res;
  //	}
  //	  
  //
  //  public double[] scoreSignature(StateSet stateSet, int tag) {
  //		return scoreSignature(stateSet.wordIndex, stateSet.sigIndex, tag);
  //  }
  //
  //	
  //	public double[] scoreSignature(String word, String sig, int tag) {
  //		int globalWordIndex = wordIndexer.indexOf(word);
  //		int globalSigIndex = wordIndexer.indexOf(sig);
  //		return scoreSignature(globalWordIndex, globalSigIndex, tag);
  //  }
  //  	
  //	public double[] scoreSignature(int globalWordIndex, int globalSigIndex, int tag) {
  //		if (globalWordIndex>=0 && wordCounter[globalWordIndex]>knownWordCount) return null;
  //  	double[] res = new double[numSubStates[tag]];
  //		if (globalSigIndex!=-1) {
  //			int tagSpecificWordIndex = tagWordIndexer[tag].indexOf(globalSigIndex);
  //			if (tagSpecificWordIndex!=-1){
  //				for (int i=0; i<numSubStates[tag]; i++){
  //					res[i] += scores[tag][i][tagSpecificWordIndex]; 
  //				}
  //			}
  //		} else{
  //			System.out.println("unseen sig");
  //		}
  //		return res;
  //  }
  //  
  //	public void labelTrees(StateSetTreeList trainTrees){
  //  	for (Tree<StateSet> tree : trainTrees){
  //  		List<StateSet> words = tree.getYield();
  //  		List<StateSet> tags = tree.getPreTerminalYield();
  //  		int ind = 0;
  //  		for (StateSet word : words){
  //  			word.wordIndex = (short)wordIndexer.indexOf(word.getWord());
  //  			short tag = tags.get(ind).getState();
  ////				if (wordIsAmbiguous[word.wordIndex]) {
  //					String sig = getSignature(word.getWord(), ind);
  //					wordIndexer.add(sig);
  //	  			word.sigIndex = (short)wordIndexer.indexOf(sig);
  //					tagWordIndexer[tag].add(wordIndexer.indexOf(sig));
  ////				}
  ////				else { word.sigIndex = -1; }
  //				ind++;
  //  		}  		
  //  	}
  //
  //	}
  //	
  //  public void init(StateSetTreeList trainTrees){
  //  	for (Tree<StateSet> tree : trainTrees){
  //  		List<StateSet> words = tree.getYield();
  //  		List<StateSet> tags = tree.getPreTerminalYield();
  //  		int ind = 0;
  //  		for (StateSet word : words){
  //				String sig = word.getWord();
  //				wordIndexer.add(sig);
  //				tagWordIndexer[tags.get(ind).getState()].add(wordIndexer.indexOf(sig));
  //  			word.wordIndex = (short)wordIndexer.indexOf(sig);
  //				ind++;
  //  		}
  //  	}
  //  	wordCounter = new int[wordIndexer.size()];
  //  	tagWordIndexer = new IntegerIndexer[numStates];
  //  	for (int tag=0; tag<numStates; tag++){
  //  		tagWordIndexer[tag] = new IntegerIndexer(wordIndexer.size());
  //  	}
  ////  	int[] firstTag = new int[wordIndexer.size()];
  //////  	wordIsAmbiguous = new boolean[wordIndexer.size()];
  ////  	for (Tree<StateSet> tree : trainTrees){
  ////  		List<StateSet> words = tree.getYield();
  ////  		List<StateSet> tags = tree.getPreTerminalYield();
  ////  		int ind = 0;
  ////  		for (StateSet word : words){
  ////  			short tag = tags.get(ind).getState();
  ////				ind++;
  ////				if (firstTag[word.wordIndex]==0) firstTag[word.wordIndex] = tag;
  ////				else if (firstTag[word.wordIndex] != tag) {
  //////					wordIsAmbiguous[word.wordIndex] = true;
  ////				}
  ////  		}
  ////  	}
  //  	labelTrees(trainTrees);
  //  	expectedCounts = new double[numStates][][];
  //  	scores = new double[numStates][][];
  //  	for (int tag=0; tag<numStates; tag++){
  //  		expectedCounts[tag] = new double[numSubStates[tag]][tagWordIndexer[tag].size()];
  //  		scores[tag] = new double[numSubStates[tag]][tagWordIndexer[tag].size()];
  //  	}
  //  	nWords = wordIndexer.size();
  //  }
  //  
  //
  //
  // }
  