package edu.wust.darcy.framework.ik;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StringReader;

import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.wltea.analyzer.IKSegmentation;
import org.wltea.analyzer.Lexeme;
import org.wltea.analyzer.lucene.IKAnalyzer;

public class IKAnalyzerTest {
	public static void main(String[] args) throws IOException {
		String filePath = "/Users/yuanzhongming/git/letsseo/src/test/resources/test.txt";
		String news = new String();
		BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(filePath), "UTF8"));
		String str;
		while ((str = in.readLine()) != null) {
			news += str;
		}
		in.close();

		System.out.println(news);
		IKAnalyzer analyzer = new IKAnalyzer(true);
		StringReader reader = new StringReader(news);
		TokenStream ts = analyzer.tokenStream("", reader);
		CharTermAttribute term = ts.getAttribute(CharTermAttribute.class);
		while (ts.incrementToken()) {
			System.out.print(term.toString() + "|");
		}
		analyzer.close();
		reader.close();

		System.out.println();
		StringReader re = new StringReader(news);
		IKSegmentation ik = new IKSegmentation(re, true);
		Lexeme lex = null;
		while ((lex = ik.next()) != null) {
			System.out.print(lex.getLexemeText() + "|");
		}
	}
}