clueweb-master/src/main/java/org/clueweb/data/TermStatistics.java
/*
 * ClueWeb Tools: Hadoop tools for manipulating ClueWeb collections
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License. You may
 * obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

package org.clueweb.data;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.WritableUtils;
import org.clueweb.clueweb12.app.BuildDictionary;

import com.google.common.base.Preconditions;

public class TermStatistics {
  private final int numTerms;
  private final long[] cfs;
  private final int[] dfs;

  private long collectionSize;

  private long maxCf = 0;
  private int maxCfTerm;

  private int maxDf = 0;
  private int maxDfTerm;

  /**
   * Creates a {@code CfTable} object.
   *
   * @param file collection frequency data file
   * @throws IOException
   */
  public TermStatistics(Path file) throws IOException {
    this(file, FileSystem.get(new Configuration()));
  }

  /**
   * Creates a {@code CfTable} object.
   *
   * @param file collection frequency data file
   * @param fs FileSystem to read from
   * @throws IOException
   */
  public TermStatistics(Path file, FileSystem fs) throws IOException {
    Preconditions.checkNotNull(file);
    Preconditions.checkNotNull(fs);

    FSDataInputStream in = fs.open(new Path(file, BuildDictionary.CF_BY_ID_DATA));
    this.numTerms = in.readInt();

    cfs = new long[numTerms];

    for (int i = 0; i < numTerms; i++) {
      long cf = WritableUtils.readVLong(in);

      cfs[i] = cf;
      collectionSize += cf;

      if (cf > maxCf) {
        maxCf = cf;
        maxCfTerm = i + 1;
      }
    }

    in.close();

    in = fs.open(new Path(file, BuildDictionary.DF_BY_ID_DATA));
    if (numTerms != in.readInt() ) {
      throw new IOException("df data and cf data should have the same number of entries!");
    }

    dfs = new int[numTerms];

    for (int i = 0; i < numTerms; i++) {
      int df = WritableUtils.readVInt(in);

      dfs[i] = df;

      if (df > maxDf) {
        maxDf = df;
        maxDfTerm = i + 1;
      }
    }

    in.close();
  }

  public int getDf(int term) {
    if (term <= 0 || term > numTerms) {
      return 0;
    }
    return dfs[term - 1];
  }

  public long getCf(int term) {
    if (term <= 0 || term > numTerms) {
      return 0;
    }

    return cfs[term - 1];
  }

  public long getCollectionSize() {
    return collectionSize;
  }

  public int getVocabularySize() {
    return numTerms;
  }

  public int getMaxDf() {
    return maxDf;
  }

  public long getMaxCf() {
    return maxCf;
  }

  public int getMaxDfTerm() {
    return maxDfTerm;
  }

  public int getMaxCfTerm() {
    return maxCfTerm;
  }

}
