package com.quui.tm2.agents;

import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Scanner;
import java.util.regex.MatchResult;

import com.quui.tm2.Agent;
import com.quui.tm2.Annotation;
import com.quui.tm2.ImmutableAnnotation;

/**
 * The tokenizer splits the text into discrete tokens using a regular expression which supports Unicode-sensitive word
 * splitting.
 */
public class Tokenizer implements Agent<String, String> {
  
  public Tokenizer(String s){
    
  }
  public Tokenizer(){
    
  }

    enum Pattern {
        /** International in-word alphanumeric letter sign. */
        LETTER("\\p{L}"),
        /** Split at one or more non-letter signs. */
        DELIM("[^" + LETTER.val + "]+"),
        /** A word: one or more in-word letters. */
        WORD(LETTER.val + "+");
        String val;

        @Override
        public String toString() {
            return String.format("@%s=%s@", this.name(), this.val);
        }

        Pattern(String pattern) {
            this.val = pattern;
        }
    }

    /**
     * {@inheritDoc}
     * @see com.quui.tm2.Agent#process(java.util.List)
     */
    public List<Annotation<String>> process(List<Annotation<String>> text) {
        List<Annotation<String>> writtenAnnotations = new ArrayList<Annotation<String>>();
        Scanner s = new Scanner(text.get(0).getValue());
        s.useDelimiter(Pattern.DELIM.val);
        while (s.hasNext(Pattern.WORD.val)) {
            MatchResult result = s.match();
            Annotation<String> a = ImmutableAnnotation.getInstance(getClass(), s.next(Pattern.WORD.val), BigInteger
                    .valueOf(result.start()), BigInteger.valueOf(result.end()));
            writtenAnnotations.add(a);
        }
        return writtenAnnotations;
    }

    /**
     * {@inheritDoc}
     * @see java.lang.Object#toString()
     */
    @Override
    public String toString() {
        return getClass().getSimpleName() + " " + Arrays.asList(Pattern.values());
    }
}
