THUCTC-master/src/org/thunlp/text/classifiers/LiblinearTextClassifier.java
package org.thunlp.text.classifiers;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Hashtable;
import java.util.Map;
import java.util.PriorityQueue;
import java.util.Vector;

import libsvm.svm;
import libsvm.svm_model;
import libsvm.svm_node;
import libsvm.svm_parameter;
import libsvm.svm_problem;

import de.bwaldvogel.liblinear.*;
import de.bwaldvogel.*;

import org.apache.commons.codec.binary.Base64;
import org.thunlp.io.TextFileWriter;
import org.thunlp.language.chinese.LangUtils;
import org.thunlp.language.chinese.WordSegment;
import org.thunlp.text.DocumentVector;
import org.thunlp.text.Lexicon;
import org.thunlp.text.Term;
import org.thunlp.text.TfIdfTermWeighter;
import org.thunlp.text.TfOnlyTermWeighter;
import org.thunlp.text.Lexicon.Word;

public abstract class LiblinearTextClassifier implements TextClassifier{
	public  Lexicon lexicon; // 词典
	private DocumentVector trainingVectorBuilder; // 用来构造训练特征向量
	private DocumentVector testVectorBuilder; // 用来构造待分类文本的特征向量
	private WordSegment seg;
	//private svm_model model; // 训练好的模型
	private de.bwaldvogel.liblinear.Model lmodel;
	private int maxFeatures = 5000; // 默认的最大特征数
	private int nclasses; // 类别数
	private int longestDoc; // 最长的文档向量长度，决定读取临时文件时缓冲大小
	private int ndocs; //训练集的大小

	public ArrayList<Integer> labelIndex = new ArrayList<Integer>(); // 类别标签
	public File tsCacheFile; // 训练集的cache文件，存放在磁盘上
	public DataOutputStream tsCache = null; // 训练集的cache输出流
	
	public int getLongestDoc() {
		return longestDoc;
	}
	
	public void init ( int nclasses, WordSegment seg) {
		lexicon = new Lexicon();
		trainingVectorBuilder =
		  new DocumentVector(lexicon, new TfOnlyTermWeighter());
		testVectorBuilder = null;
		//model = null;
		lmodel = null;
		this.nclasses = nclasses;
		ndocs = 0;
		this.seg = seg;
	}
	
	public Lexicon getLexicon() {
		return lexicon;
	}

	
	public void clear () {
		lexicon = null;
		trainingVectorBuilder = null;
		testVectorBuilder = null;
		lmodel = null;
		seg = null;
		labelIndex = null;
	}
	
	abstract protected WordSegment initWordSegment();

	public LiblinearTextClassifier( int nclasses ) {
		init( nclasses, initWordSegment());
	}
	
	/**
	 * 初始化一个基于bigram和svm的中文文本分类器
	 * @param nclasses 类别数
	 */
	public LiblinearTextClassifier( int nclasses, WordSegment seg ) {
		init( nclasses, seg);
	}
	
	/**
	 * 利用Scalable Term Selection方法进行特征选择
	 * @author sames
	 * @param cacheFile 数据集，其中term的weight应该是tf
	 * @param featureSize 数据集中特征的总数，特征应该是从0到featureSize编号
	 * @param kept 需要保留的特征数
	 * @param ndocs 训练集文档数
	 * @param nclasses 类别数
	 * @param longestDoc 最长文档含有的特征数
	 * @return
	 */
	public Map<Integer, Integer> selectFeatureBySTS(File cacheFile,
			int featureSize, int kept, int ndocs, int nclasses, int longestDoc) {
		
		// lamda初始值
		double lamda = 0.5;
		int[][] featureStats = new int[featureSize][nclasses];
		int[] featureFreq = new int[featureSize];
		double[] prValues = new double[featureSize];
		PriorityQueue<Term> selectedFeatures;
		int[] classSize = new int[nclasses];

		// 统计chi-square需要的计数
		int label = 0;
		int nterms = 0;
		double sum = 0;
		Term[] terms = new Term[longestDoc + 1];
		for (int i = 0; i < terms.length; i++) {
			terms[i] = new Term();
		}
		int ndocsread = 0;
		try {
			DataInputStream dis = new DataInputStream(new BufferedInputStream(
					new FileInputStream(cacheFile)));
			while (true) {
				int ncut = 0;
				try {
					label = dis.readInt();
					nterms = dis.readInt();
					sum += nterms;
					for (int i = 0; i < nterms; i++) {
						terms[i].id = dis.readInt();
						terms[i].weight = dis.readDouble();
						if ( lexicon.getWord(terms[i].id).getDocumentFrequency()
								== 1 ) 
							ncut++;
					}
				} catch (EOFException e) {
					break;
				}
				sum -= ncut;

				classSize[label]++;
				for (int i = 0; i < nterms; i++) {
					Term t = terms[i];
					featureStats[t.id][label]++;
					featureFreq[t.id]++;
				}				
			}
			dis.close();
		} catch (IOException e) {
			return null;
		}

		System.err.println("start STS calculation");

		// 计算chi^2_avg(t)，这里利用一个优先级队列来选择chi^2最高的特征
		selectedFeatures = new PriorityQueue<Term>(kept + 1,
				new Comparator<Term>() {
					public int compare(Term t0, Term t1) {
						return Double.compare(t0.weight, t1.weight);
					}
				});

		long A, B, C, D;

		for (int i = 0; i < featureSize; i++) {
			double pr = -1;
			double prmax = -1;
			for (int j = 0; j < nclasses; j++) {
				A = featureStats[i][j];
				B = featureFreq[i] - A;
				C = classSize[j];
				D = ndocs - C;

				double fractorBase = (double) (B * C);
				if (Double.compare(fractorBase, 0.0) == 0) {
					pr = Double.MAX_VALUE;
				} else {
					pr = (double) (A * D) / fractorBase;
					if (pr > prmax) {
						prmax = pr;
						prValues[i] = prmax;
					}
				}
			}
		}

		double targetAVL = Math.pow(sum / ndocs, 0.085 * Math.log(kept));

		Term[] featuresToSort = new Term[kept];

		double first = 0;
		double second = 1;
		int iteration = 1;
		while (true) {
			selectedFeatures.clear();
			for (int i = 0; i < featureSize; i++) {
				if ( lexicon.getWord(i).getDocumentFrequency() == 1 )
					continue;
				Term t = new Term();
				t.id = i;
				t.weight = 1.0 / (lamda / Math.log(prValues[i]) + (1 - lamda)
						/ Math.log(featureFreq[i]));
				selectedFeatures.add(t);
				if (selectedFeatures.size() > kept) {
					selectedFeatures.poll();
				}
			}
			double AVL = 0;
			int n = 0;
			while (selectedFeatures.size() > 0) {
				Term t = selectedFeatures.poll();
				featuresToSort[n] = t;
				n++;
				AVL += featureFreq[t.id];
			}
			Arrays.sort(featuresToSort, new Term.TermIdComparator());
			AVL /= ndocs;

			System.out.println("Iteration:" + iteration + " lamda = " + lamda
					+ " AVL = " + AVL + " Target AVL = " + targetAVL);
			if (Math.abs(AVL - targetAVL) < 0.1)
				break;
			else {
				if (AVL < targetAVL) {
					second = lamda;
					lamda = (first + lamda) / 2;					
				} else {
					first = lamda;
					lamda = (lamda + second) / 2;				
				}
				if( Math.abs(second - first) < 1.0E-13 )
					break;				
			}
			iteration ++;
		}

		System.err.println("generating feature map");

		// 生成旧id和新选择的id的对应表
		Map<Integer, Integer> fidmap = new Hashtable<Integer, Integer>(kept);

		for (int i = 0; i < featuresToSort.length; i++) {
			fidmap.put(featuresToSort[i].id, i);

		}
		return fidmap;

	}

	
	/**
	 * 利用chi-square统计量来进行特征选择
	 * @param dataSet 数据集，其中term的weight应该是tf
	 * @param featureSize 数据集中特征的总数，特征应该是从0到featureSize编号的
	 * @param kept 要保留的特征数
	 * @return 选择前特征到选择后特征的id对应表，保证选择后特征的排序和选择前一样
	 */
	public Map<Integer, Integer> selectFeaturesByChiSquare( 
			File cacheFile, 
			int featureSize,
			int kept ) {
		return selectFeaturesByChiSquare(cacheFile, featureSize, kept, 
				ndocs, nclasses, longestDoc, null);
	}
	
	/**
	 * 真正的特征选择函数，允许输出所有特征chimax的值，用于调试
	 * @param cacheFile
	 * @param featureSize
	 * @param kept
	 * @param chimaxValues 每个特征的chimax值，如果为null则不记录
	 * @return
	 */
	public Map<Integer, Integer> selectFeaturesByChiSquare( 
			File cacheFile, 
			int featureSize,
			int kept, 
			int ndocs,
			int nclasses,
			int longestDoc,
			double [] chimaxValues ) {
		
		System.out.println("selectFeatureBySTS : " +
				"featureSize = " + featureSize + 
				"; kept = " + kept + 
				"; ndocs = " + ndocs + 
				"; nclasses = " + nclasses + 
				"; longestDoc = " + longestDoc);
		
		int [][] featureStats = new int[featureSize][nclasses]; //某词在某类出现次数
		int [] featureFreq = new int[featureSize];//某词词频
		PriorityQueue<Term> selectedFeatures;
		int [] classSize = new int[nclasses];//每类多少篇文章
		
		// 统计chi-square需要的计数
		int label = 0;
		int nterms = 0;
		Term [] terms = new Term[longestDoc + 1];
		for ( int i = 0 ; i < terms.length ; i++ ) {
			terms[i] = new Term();
		}
		int ndocsread = 0;
		try {
			DataInputStream dis = new DataInputStream(new BufferedInputStream(
					new FileInputStream(cacheFile)));
			while ( true ) {
				try {
					label = dis.readInt();
					nterms = dis.readInt();
					//System.out.println("Reading doc "+ ndocsread + " :  label = " + label + ";  nterms = " + nterms);
					for ( int i = 0 ; i < nterms ; i++ ) { 
						terms[i].id = dis.readInt();
						terms[i].weight = dis.readDouble();
					}
				} catch ( EOFException e ) {
					break;
				}
				
				classSize[label] ++;
				for ( int i = 0 ; i < nterms ; i++ ) { 
					Term t = terms[i];
					featureStats[t.id][label] ++;
					featureFreq[t.id] ++;
				}
				if ( ndocsread++ % 10000 == 0) {
					System.err.println("scanned " + ndocsread);
				}
			}
			dis.close();
		} catch ( IOException e ) {
			return null;
		}
		
		
		System.err.println("start chi-square calculation");
		
		// 计算chi^2_avg(t)，这里利用一个优先级队列来选择chi^2最高的特征
		selectedFeatures = new PriorityQueue<Term>( kept + 1, 
				new Comparator<Term>() {
					public int compare(Term t0, Term t1) {
						return Double.compare(t0.weight, t1.weight);
					}			
		});
		
		long A, B, C, D;
		
		for ( int i = 0 ; i < featureSize ; i++ ) {
			Word w = lexicon.getWord(i);
			if (w != null) {
			  if ( w.getDocumentFrequency() == 1 || w.getName().length() > 50 )
			    continue;
			}
			double chisqr = -1;
			double chimax = -1;
			for ( int j = 0 ; j < nclasses ; j++ ) {
				A = featureStats[i][j];
				B = featureFreq[i] - A;
				C = classSize[j] - A;
				D = ndocs - A - B - C;
				
				//System.out.println("A:"+A+" B:"+B+" C:"+C+" D:"+D);
				double fractorBase = (double)( (A+C) * (B+D) * (A+B) * (C+D) );
				if ( Double.compare(fractorBase, 0.0 ) == 0 ) {
					chisqr = 0;
				} else {
					// 我们不用ndocs，因为所有特征的ndocs都一样
					//chisqr = ndocs * ( A*D -B*C) * (A*D - B*C) / fractorBase  ;
					chisqr = ( A*D -B*C) / fractorBase * (A*D - B*C)   ;
				}
				if ( chisqr > chimax ) {
					chimax = chisqr;
				}

//				被注释的方法是计算chi^2_avg即概率加权平均的卡方值。我们实际用的是chimax
//				chisqr += (classSize[j] / (double) ndocs) * 
//						ndocs * ( A*D -B*C) * (A*D - B*C) 
//						/ (double)( (A+C) * (B+D) * (A+B) * (C+D) ) ;
			}
			if ( chimaxValues != null ) {
				chimaxValues[i] = chimax;
			}
			Term t = new Term();
			t.id = i;
			t.weight = chimax;
			selectedFeatures.add(t);
			if ( selectedFeatures.size() > kept ) {
				selectedFeatures.poll();
			}
		}
		outputSecletedFeatures(selectedFeatures);
		System.err.println("generating feature map");
		
		// 生成旧id和新选择的id的对应表
		Map<Integer, Integer> fidmap = new Hashtable<Integer, Integer>(kept);
		Term [] featuresToSort = new Term[selectedFeatures.size()];
		int n = 0;
		while ( selectedFeatures.size() > 0 ) {
			Term t = selectedFeatures.poll();
			featuresToSort[n] = t;
			n++;
		}
		Arrays.sort(featuresToSort, new Term.TermIdComparator());
		for ( int i = 0 ; i < featuresToSort.length ; i++ ) {
			fidmap.put(featuresToSort[i].id, i);
		}
		return fidmap;
	}
	
	public void outputSecletedFeatures(PriorityQueue<Term> features){
		System.out.println("store features...=======================================");
		try{
			TextFileWriter tw = new TextFileWriter("selectedFeatures","UTF-8");
			Term[] f;
			f = features.toArray(new Term[features.size()]);
			System.out.println(f.length);
			for(int i=f.length-1; i>=0; i--){
				tw.writeLine(lexicon.getWord(f[i].id).getName() + " " + f[i].weight);
			}
			tw.flush();
			tw.close();
			
		}catch(IOException e){
			e.printStackTrace();
		}
		System.out.println("end store features...=======================================");
	}
	
	public void setMaxFeatures( int max ) {
		maxFeatures = max;
	}
	
	public int getMaxFeatures() {
		return maxFeatures;
	}
	

	/**
	 * 加入一篇训练文档。要求label是小于总类别数的整数，从0开始。
	 * @param text 训练文本
	 * @param label 类别编号
	 * @return 加入是否成功。不成功可能是由于不能在磁盘上创建临时文件
	 */
	public boolean addTrainingText(String text, int label) {
		if ( label >= nclasses || label < 0 ) {
			return false;
		}
		if ( tsCache == null ) {
			try {
				//tsCacheFile = File.createTempFile("tctscache", "data");
				tsCacheFile = new File(".", "tctscache" + Long.toString(System.currentTimeMillis()) + "data");
				tsCache = new DataOutputStream(
								new BufferedOutputStream(
								new FileOutputStream(tsCacheFile)));
				longestDoc = 0;
			} catch (IOException e) {
				return false;
			}
		}
		text = LangUtils.removeEmptyLines(text);
		text = LangUtils.removeExtraSpaces(text);
		String [] bigrams = seg.segment(text);
		lexicon.addDocument(bigrams);
		Word [] words = lexicon.convertDocument(bigrams);
		bigrams = null;
		Term [] terms = trainingVectorBuilder.build( words, false );
		try {
			tsCache.writeInt(label);
			tsCache.writeInt(terms.length);
			if ( terms.length > longestDoc ) {
				longestDoc = terms.length;
			}
			for ( int i = 0 ; i < terms.length ; i++ ) {
				tsCache.writeInt(terms[i].id);
				tsCache.writeDouble(terms[i].weight);
			}
		} catch (IOException e) {
			return false;
		}
		if ( ! labelIndex.contains(label) ) {
			labelIndex.add(label);
		}
		ndocs++;
		return true;
	}

	/**
	 * 分类一篇文档
	 * @param text 待分类文档
	 * @return 分类结果，其中包含分类标签和概率，对于svm分类器，概率无意义
	 */

	public ClassifyResult classify(String text) {
		String [] bigrams = seg.segment(text);
		Word [] words = lexicon.convertDocument(bigrams);
		bigrams = null;
		Term [] terms = testVectorBuilder.build( words, true);
		
		int m = terms.length;
		//svm_node[] x = new svm_node[m];
		FeatureNode[] lx = new FeatureNode[m];
		for(int j = 0; j < m; j++)
		{
			lx[j] = new FeatureNode(terms[j].id + 1, terms[j].weight);
		}
		
		ClassifyResult cr = new ClassifyResult(-1, -Double.MAX_VALUE);
		//double [] probs = new double[svm.svm_get_nr_class(model)];
		double[] probs = new double[this.lmodel.getNrClass()];
		//svm.svm_predict_probability(model, x, probs);
		//de.bwaldvogel.liblinear.Linear.predictValues(lmodel, lx, probs);
		de.bwaldvogel.liblinear.Linear.predictProbability(lmodel, lx, probs);
		for (int i = 0; i < probs.length; i++) {
		  if (probs[i] > cr.prob) {
		    cr.prob = probs[i];
		    cr.label = i;
		  }
		}
		return cr;
	}
	
	public ClassifyResult[] classify(String text, int topN){
		
		String [] bigrams = seg.segment(text);
		Word [] words = lexicon.convertDocument(bigrams);
		bigrams = null;
		Term [] terms = testVectorBuilder.build(words, true);
		
		int m = terms.length;
		//svm_node[] x = new svm_node[m];
		FeatureNode[] lx = new FeatureNode[m];
		for(int j = 0; j < m; j++)
		{
			lx[j] = new FeatureNode(terms[j].id + 1, terms[j].weight);
		}
		
		//double [] probs = new double[svm.svm_get_nr_class(model)];
		double[] probs = new double[this.lmodel.getNrClass()];
		ArrayList<ClassifyResult> cr = new ArrayList<ClassifyResult>();
		//svm.svm_predict_probability(model, x, probs);
		//de.bwaldvogel.liblinear.Linear.predictValues(lmodel, lx, probs);
		de.bwaldvogel.liblinear.Linear.predictProbability(lmodel, lx, probs);
		for(int i=0; i<probs.length; i++){
			cr.add(new ClassifyResult(i, probs[i]));
		}
		Comparator com = new Comparator() {
			public int compare(Object obj1, Object obj2){
				ClassifyResult o1 = (ClassifyResult)obj1;
				ClassifyResult o2 = (ClassifyResult)obj2;
				if(o1.prob > o2.prob + 1e-20) return -1;
				else if(o1.prob < o2.prob - 1e-20) return 1;
				else return 0;
			}
		};
		
		Collections.sort(cr,com);
		
		/*
		double totalexp = 0.0;
		for(int i=0; i<probs.length;i++){
			totalexp += Math.exp(probs[i]);
		}
		for(int i=0; i<topN; i++){
			results[i] = ""+ al.get(i).index +" "+ Math.exp(al.get(i).value)/totalexp;
		}*/
		java.text.DecimalFormat dcmFmt = new DecimalFormat("0.0000");
		ClassifyResult result[] = new ClassifyResult[topN];
		for(int i=0; i<topN; i++){
			result[i] = new ClassifyResult(cr.get(i).label, cr.get(i).prob);
		}
		cr.clear();
		//System.out.println(""+totalexp+results[0]);
		return result;
	}
	
	
	public ClassifyResult[] classify(String text, String mode){
		
		/*
		double mean1 = 0.9587235538481998;
		double mean2 = 0.027685258283684743;
		double mean3 = 0.0047742850704747064;
		double sd1 = 0.11380367342131785;
		double sd2 = 0.07759971712653867;
		double sd3 = 0.02049807196764366;
		*/
		
		double mean1 = 0.8976;
		double mean2 = 0.0663;
		double mean3 = 0.0106;
		double sd1 = 0.1547;
		double sd2 = 0.1165;
		double sd3 = 0.0275;
		
		String [] bigrams = seg.segment(text);
		Word [] words = lexicon.convertDocument(bigrams);
		bigrams = null;
		Term [] terms = testVectorBuilder.build( words, true);
		
		
		int m = terms.length;
		FeatureNode[] lx = new FeatureNode[m];
		for(int j = 0; j < m; j++){
			lx[j] = new FeatureNode(terms[j].id + 1, terms[j].weight);
		}
		
		double[] probs = new double[this.lmodel.getNrClass()];
		//de.bwaldvogel.liblinear.Linear.predictValues(lmodel, lx, probs);
		de.bwaldvogel.liblinear.Linear.predictProbability(lmodel, lx, probs);
		
		ArrayList<ClassifyResult> al = new ArrayList<ClassifyResult>();
		for(int i=0; i<probs.length; i++){
			al.add(new ClassifyResult(i, probs[i]));
		}
		if(al.size() ==0){
			System.err.println("error!result size is 0!");
			return null;
		}
		Comparator com = new Comparator() {
			public int compare(Object obj1, Object obj2){
				ClassifyResult o1 = (ClassifyResult)obj1;
				ClassifyResult o2 = (ClassifyResult)obj2;
				if(o1.prob > o2.prob + 0.0000000001) return -1;
				else if(o1.prob <o2.prob - 0.0000000001) return 1;
				else return 0;
			}
		};
		
		Collections.sort(al,com);
		
		int num=0;
		ArrayList<ClassifyResult> res = new ArrayList<ClassifyResult>();
		
		java.text.DecimalFormat dcmFmt = new DecimalFormat("0.0000");
		
		res.add(new ClassifyResult(al.get(0).label, al.get(0).prob));
		if(al.size() >=3){
			if(al.get(1).prob-mean2>2*sd2/3){
				res.add(new ClassifyResult(al.get(1).label, al.get(1).prob));
				if(al.get(0).prob+al.get(1).prob<0.99){
					if(al.get(2).prob-mean3>sd3){
						res.add(new ClassifyResult(al.get(2).label, al.get(2).prob));
					}
				}
			}
		}
		return res.toArray(new ClassifyResult[res.size()]);
		//System.out.println(""+totalexp+results[0]);
		//return results;
	}
	/**
	 * 从磁盘上加载训练好的模型
	 * @param filename 模型文件名(是一个目录)
	 * @return 加载是否成功
	 */
	
	public boolean loadModel(String filename) {
		File modelPath = new File(filename);
		if ( ! modelPath.isDirectory() )
			return false;
		
		File lexiconFile = new File( modelPath, "lexicon");
		File modelFile = new File( modelPath, "model");
		
		System.out.println(lexiconFile.getAbsolutePath());
		
		try { 
			if ( lexiconFile.exists() ) {
				lexicon.loadFromFile(lexiconFile);
				System.out.println("lexicon exists!");
			} else {
				return false;
			}
	
			if ( modelFile.exists() ) {
				//this.model = svm.svm_load_model(modelFile.getAbsolutePath());
				//this.lmodel = de.bwaldvogel.liblinear.Linear.loadModel(new File(modelFile.getAbsolutePath()));
				System.out.println("model exists!");
				this.lmodel = de.bwaldvogel.liblinear.Linear.loadModel(modelFile);
			} else {
				return false;
			}
		} catch ( Exception e ) {
			return false;
		}
		lexicon.setLock( true );
		trainingVectorBuilder = null;
		testVectorBuilder =
		  new DocumentVector(lexicon, new TfIdfTermWeighter(lexicon));
		return true;
	}
	/**
	 * 将训练好的模型保存到磁盘
	 * @param filename 保存的文件名(实际是一个目录)
	 * @return 保存是否成功
	 */

	public boolean  saveModel(String filename) {
		File modelPath = new File(filename);
		if (!modelPath.exists() && !modelPath.mkdir() ) {
			return false;
		}
		
		File lexiconFile = new File( modelPath, "lexicon");
		File modelFile = new File( modelPath, "model");
		
		try {
			lexicon.saveToFile(lexiconFile);
			//svm.svm_save_model(modelFile.getAbsolutePath(), model);
			de.bwaldvogel.liblinear.Linear.saveModel(new File(modelFile.getAbsolutePath()), lmodel);
		} catch (IOException e ) {
			return false;
		}
		return true;
	}
	/**
	 * 训练模型
	 * @return 训练是否成功。不成功可能是由于不能正确地读写临时文件造成的
	 */
	public boolean train() {
		try {
			tsCache.close();
		} catch (IOException e) {
			return false;
		}
		
		Map<Integer, Integer> selectedFeatures = selectFeaturesByChiSquare(
				tsCacheFile, lexicon.getSize(), maxFeatures);
		
		//以下注释的代码为用李景阳论文Scalable Term Selection方法选择特征，目前未经完全测试通过！！
//		Map<Integer, Integer> selectedFeatures = selectFeatureBySTS(
//				tsCacheFile, lexicon.getSize(), maxFeatures, ndocs, nclasses,
//				longestDoc);
		if ( selectedFeatures == null ) {
			return false;
		}
		System.err.println("feature selection complete");
		//svm_problem problem = createLibSVMProblem(tsCacheFile, selectedFeatures);
		///////////////////add
		de.bwaldvogel.liblinear.Problem lproblem = createLiblinearProblem(tsCacheFile, selectedFeatures);
		System.err.println("liblinear problem created");
		
		lexicon = lexicon.map( selectedFeatures );
		lexicon.setLock( true );
		tsCacheFile.delete();
		trainingVectorBuilder = null;
		testVectorBuilder = new DocumentVector(lexicon, new TfIdfTermWeighter(lexicon));

		de.bwaldvogel.liblinear.Parameter lparam = new Parameter(SolverType.L1R_LR, 500, 0.01);
		//de.bwaldvogel.liblinear.Parameter lparam = new Parameter(solverType, C, eps)

		de.bwaldvogel.liblinear.Model tempModel = de.bwaldvogel.liblinear.Linear.train(lproblem, lparam);
		System.err.println("TRAINING COMPLETE=========================================================================================");
		this.lmodel = tempModel;
		//this.model = (svm_model)tempModel;
		return true;
	}
	
	private static class DataNode implements Comparable{
		int label;
		svm_node [] nodes;
		
		public int compareTo( Object o ) {
			DataNode other = (DataNode) o;
			return label - other.label;
		}
	}
	
	private static class LdataNode implements Comparable{
		int llabel;
		FeatureNode [] lnodes;
		
		public int compareTo( Object o ) {
			LdataNode other = (LdataNode) o;
			return llabel - other.llabel;
		}
	}
	
	private de.bwaldvogel.liblinear.Problem createLiblinearProblem( File cacheFile, 
			Map<Integer, Integer> selectedFeatures){
		Vector<Double> vy = new Vector<Double>();
		Vector<svm_node[]> vx = new Vector<svm_node[]>();
		
		//DataNode [] datanodes = new DataNode[this.ndocs];
		LdataNode [] ldatanodes = new LdataNode[this.ndocs];
		//FeatureNode[][] lfeatureNodes;
		
		int label, nterms;
		Term [] terms = new Term[longestDoc + 1];
		for ( int i = 0 ; i < terms.length ; i++ ) {
			terms[i] = new Term();
		}
		int ndocsread = 0;
		//add------------------------
		int maxIndex=0;
		try {
			DataInputStream dis = new DataInputStream(new BufferedInputStream(
					new FileInputStream(cacheFile)));
			while ( true ) {
				int n = 0;
				try {
					label = dis.readInt();
					nterms = dis.readInt();
					for ( int i = 0 ; i < nterms ; i++ ) { 
						int tid = dis.readInt();
						double tweight = dis.readDouble();
						Integer id = selectedFeatures.get(tid);
						if ( id != null ) {
							terms[n].id = id;
							//add
							maxIndex=Math.max(maxIndex, id+1);
							
							Word w = lexicon.getWord(tid);
							int df = w.getDocumentFrequency();
							terms[n].weight = Math.log( tweight + 1 ) 
								* ( Math.log( (double) ( ndocs + 1 ) / df ) );
							n++;
							//System.err.println("doc " + id + " " + w);
						}
					}
				} catch ( EOFException e ) {
					break;
				}
				
				
				//----------------------------
				//lfeatureNodes = new FeatureNode[this.ndocs][n];
				//-----------------------
				
				//System.out.println("===================================================n: "+n);
				// 归一化向量
				double normalizer = 0;
				for ( int i = 0 ; i < n ; i++ ) { 
					normalizer += terms[i].weight * terms[i].weight;
				}
				normalizer = Math.sqrt(normalizer);
				for ( int i = 0 ; i < n ; i++ ) { 
					terms[i].weight /= normalizer;
				}
				
				//datanodes[ndocsread] = new DataNode();
				
				// 放入svm problem中
				ldatanodes[ndocsread] = new LdataNode();
				ldatanodes[ndocsread].llabel= label;

				FeatureNode[] lx = new FeatureNode[n];
				for ( int i = 0; i < n ; i++ ) {
					lx[i] = new FeatureNode(terms[i].id + 1,terms[i].weight);
				}
				ldatanodes[ndocsread].lnodes = lx;
				
				if ( ndocsread++ % 10000 == 0) {
					System.err.println("scanned " + ndocsread);
				}
			}
			dis.close();
		} catch ( IOException e ) {
			return null;
		}
		
		assert( this.ndocs == ndocsread );
		
		Arrays.sort( ldatanodes );
		
		//svm_problem prob = new svm_problem();
		de.bwaldvogel.liblinear.Problem lprob = new de.bwaldvogel.liblinear.Problem();
		/*
		prob.l = datanodes.length;
		prob.x = new svm_node[prob.l][];
		for( int i = 0 ; i < prob.l ; i++ )
			prob.x[i] = datanodes[i].nodes;
		prob.y = new double[prob.l];
		for(int i = 0 ; i < prob.l ; i++ )
			prob.y[i] = (double) datanodes[i].label;
		
		return prob;
		*/
		//add
		System.out.println("max index: -------------------------------------: " + maxIndex);
		lprob.n = maxIndex;
		lprob.l = ldatanodes.length;
		lprob.x = new de.bwaldvogel.liblinear.FeatureNode[lprob.l][];
		for( int i = 0 ; i < lprob.l ; i++ )
			//lprob.x[i] = datanodes[i].nodes;
			lprob.x[i]=ldatanodes[i].lnodes;
		lprob.y = new int[lprob.l];
		for(int i = 0 ; i < lprob.l ; i++ )
			lprob.y[i] = ldatanodes[i].llabel;
		
		return lprob;
	}
	
	/**
	 * 根据特征选择的结果来生成一个用于训练的SVM problem
	 * @param cacheFile 存放训练集的缓存文件
	 * @param selectedFeatures 特征选择的结果
	 * @return 构造好的svm_problem数据结构
	 */

	public String saveToString() {
	  ByteArrayOutputStream baos = new ByteArrayOutputStream();
	  try {
	    ObjectOutputStream oos = new ObjectOutputStream(baos);
	    oos.writeObject(this.lexicon);
	    //oos.writeObject(this.model);
	    oos.writeObject(this.lmodel);
	    oos.close();
	  } catch (IOException e) {
	    e.printStackTrace();
	    return "";  // Failed to serialize the model.
	  }
	  String base64 = new String(Base64.encodeBase64(baos.toByteArray()));
	  return base64;
	}
	
	public void loadFromString(String model) {
	  ByteArrayInputStream bais = 
	    new ByteArrayInputStream(Base64.decodeBase64(model.getBytes()));
	  ObjectInputStream ois;
    try {
      ois = new ObjectInputStream(bais);
      this.lexicon = (Lexicon) ois.readObject();
      //this.model = (svm_model) ois.readObject();
      this.lmodel = (de.bwaldvogel.liblinear.Model) ois.readObject();
      ois.close();
    } catch (IOException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    } catch (ClassNotFoundException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    testVectorBuilder = 
      new DocumentVector(lexicon, new TfIdfTermWeighter(lexicon));
	}
}
