code: /**
   * 
   */
  package edu.berkeley.nlp.PCFGLA;
  
  import java.util.HashMap;
  
  import edu.berkeley.nlp.util.ArrayUtil;
  import edu.berkeley.nlp.util.Numberer;
  
  /**
   * @author petrov
   * 
   */
  public class GrammarAnalyzer {
  
  	public static class Options {
  		@Option(name = "-in", required = true, usage = "Grammarfile")
  		public String grFileName;
  
  		@Option(name = "-t", usage = "Threshold for pruning unlikely rules")
  		public double threshold = -1;
  	}
  
  	public static void main(String[] args) {
  		OptionParser optParser = new OptionParser(Options.class);
  		Options opts = (Options) optParser.parse(args, true);
  
  		String inFileName = opts.grFileName;
  		ParserData pData = ParserData.Load(inFileName);
  		if (pData == null) {
  			System.out.println("Failed to load grammar from file" + inFileName
  					+ ".");
  			System.exit(1);
  		}
  		Grammar grammar = pData.getGrammar();
  		Lexicon lexicon = pData.getLexicon();
  		Numberer.setNumberers(pData.getNumbs());
  
  		if (opts.threshold > -1) {
  			System.out.println("Remving rules with probability below "
  					+ opts.threshold + ".");
  			grammar.splitRules();
  			grammar.removeUnlikelyRules(opts.threshold, 1.0);
  			lexicon.removeUnlikelyTags(opts.threshold, 1.0);
  		}
  
  		int[] tagTotal = new int[grammar.numSubStates.length];
  		int[] tagOne = new int[grammar.numSubStates.length];
  
  		int[] gr = computeAndPrintCounts(grammar, tagTotal, tagOne);
  		printStats("Grammar", gr);
  		int[] lex = computeAndPrintCounts(lexicon, tagTotal, tagOne);
  		printStats("Lexicon", lex);
  
  		ArrayUtil.addInPlace(gr, lex);
  		printStats("Together", gr);
  
  		System.out.println("\nTag specific statistics (x=1):");
  		Numberer tagNumberer = Numberer.getGlobalNumberer("tags");
  		for (int i = 0; i < tagTotal.length; i++) {
  			System.out.println(tagNumberer.object(i) + ": " + tagOne[i] + "/"
  					+ tagTotal[i]);
  		}
  
  		System.exit(0);
  	}
  
  	public static int[] computeAndPrintCounts(Grammar gr, int[] tagTotal,
  			int[] tagOne) {
  		int nulledOut = 0, zero = 0, belowOne = 0, one = 0, aboveOne = 0, total = 0;
  		int unaryEqualSlices = 0, binaryEqualSlices = 0, unarySlices = 0, binarySlices = 0;
  		int unaryEqual = 0, binaryEqual = 0, unary = 0, binary = 0;
  		for (int state = 0; state < gr.numStates; state++) {
  			int nParentSubStates = gr.numSubStates[state];
  			for (UnaryRule uRule : gr.getUnaryRulesByParent(state)) {
  				int nChildSubStates = gr.numSubStates[uRule.childState];
  				total += nChildSubStates * nParentSubStates;
  				unary++;
  				double[][] scores = uRule.getScores2();
  				// boolean allTheSame=true;
  				// double veryFirst = scores[0][0];
  				for (int j = 0; j < nChildSubStates; j++) {
  					unarySlices++;
  					if (scores[j] == null) {
  						nulledOut += nParentSubStates;
  						continue;
  					}
  					boolean sliceTheSame = true;
  					double first = scores[j][0];
  					for (int i = 0; i < nParentSubStates; i++) {
  						double p = scores[j][i];
  						if (p == 0)
  							zero++;
  						else if (p == 1)
  							one++;
  						else if (p > 1)
  							aboveOne++;
  						else
  							belowOne++;
  						if (sliceTheSame && p != first)
  							sliceTheSame = false;
  						// if (allTheSame&&p!=veryFirst) allTheSame=false;
  						tagTotal[state]++;
  						if (p == 1)
  							tagOne[state]++;
  					}
  					if (sliceTheSame)
  						unaryEqualSlices++;
  				}
  				// if (allTheSame) unaryEqual++;
  			}
  			for (BinaryRule bRule : gr.splitRulesWithP(state)) {// gr.getBinaryRulesByParent(state)){
  				double[][][] scores = bRule.getScores2();
  				binary++;
  				// boolean allTheSame=true;
  				// double veryFirst = scores[0][0][0];
  				for (int j = 0; j < scores.length; j++) {
  					for (int k = 0; k < scores[j].length; k++) {
  						total += nParentSubStates;
  						binarySlices++;
  						if (scores[j][k] == null) {
  							nulledOut += nParentSubStates;
  							continue;
  						}
  						boolean sliceTheSame = true;
  						double first = scores[j][k][0];
  						for (int i = 0; i < nParentSubStates; i++) {
  							double p = scores[j][k][i];
  							if (p == 0)
  								zero++;
  							else if (p == 1)
  								one++;
  							else if (p > 1)
  								aboveOne++;
  							else
  								belowOne++;
  							if (sliceTheSame && p != first)
  								sliceTheSame = false;
  							// if (allTheSame&&p!=veryFirst) allTheSame=false;
  							tagTotal[state]++;
  							if (p == 1)
  								tagOne[state]++;
  						}
  						if (sliceTheSame)
  							binaryEqualSlices++;
  					}
  				}
  				// if (allTheSame) binaryEqual++;
  			}
  		}
  		System.out.println("Same across parent: " + unaryEqualSlices + "/"
  				+ unarySlices + " (unary), " + binaryEqualSlices + "/"
  				+ binarySlices + " (binary)");
  		System.out.println("All the same: " + unaryEqual + "/" + unary
  				+ " (unary), " + binaryEqual + "/" + binary + " (binary)");
  		return new int[] { nulledOut, zero, belowOne, one, aboveOne, total };
  	}
  
  	public static int[] computeAndPrintCounts(Lexicon lexicon, int[] tagTotal,
  			int[] tagOne) {
  		int zero = 0, belowOne = 0, one = 0, aboveOne = 0, total = 0;
  		int equal = 0, pairs = 0;
  		if (lexicon instanceof SophisticatedLexicon) {
  			SophisticatedLexicon lex = (SophisticatedLexicon) lexicon;
  			for (short tag = 0; tag < lex.numSubStates.length; tag++) {
  				HashMap<String, double[]> tagMap = lex.wordToTagCounters[tag];
  				if (tagMap == null)
  					continue;
  				for (String word : tagMap.keySet()) {
  					double[] lexiconScores = lex.score(word, tag, 0, false,
  							false);
  					for (int i = 0; i < lexiconScores.length; i++) {
  						total++;
  						double p = lexiconScores[i];
  						if (p == 0)
  							zero++;
  						else if (p == 1)
  							one++;
  						else if (p > 1)
  							aboveOne++;
  						else
  							belowOne++;
  						tagTotal[tag]++;
  						if (p == 1)
  							tagOne[tag]++;
  					}
  				}
  			}
  		} else {
  			SimpleLexicon lex = (SimpleLexicon) lexicon;
  			for (int tag = 0; tag < lex.scores.length; tag++) {
  				if (lex.tagWordIndexer[tag].size() == 0)
  					continue;
  				for (int word = 0; word < lex.scores[tag][0].length; word++) {
  					boolean allTheSame = true;
  					double first = lex.scores[tag][0][word];
  					pairs++;
  					for (int substate = 0; substate < lex.numSubStates[tag]; substate++) {
  						total++;
  						double p = lex.scores[tag][substate][word];
  						if (p == 0)
  							zero++;
  						else if (p == 1)
  							one++;
  						else if (p > 1)
  							aboveOne++;
  						else
  							belowOne++;
  						if (allTheSame && p != first)
  							allTheSame = false;
  						tagTotal[tag]++;
  						if (p == 1)
  							tagOne[tag]++;
  					}
  					if (allTheSame)
  						equal++;
  				}
  			}
  			System.out.println("Same across parent: " + equal + "/" + pairs);
  		}
  		return new int[] { 0, zero, belowOne, one, aboveOne, total };
  	}
  
  	public static void printStats(String title, int[] vals) {
  		int nulledOut = vals[0], zero = vals[1], belowOne = vals[2], one = vals[3], aboveOne = vals[4], total = vals[5];
  		System.out.println(title + " statistics:");
  		System.out.println("Total rules:\t" + total);
  		System.out.println("Total non-zero:\t"
  				+ (belowOne + one + aboveOne + "\n"));
  
  		System.out.println("Nulled out:\t" + nulledOut);
  		System.out.println("x = 0:\t\t" + zero);
  		System.out.println("0 < x < 1:\t" + belowOne);
  		System.out.println("x = 1:\t\t" + one);
  		System.out.println("x > 1:\t\t" + aboveOne + "\n");
  
  	}
  
  }
  