'Trains the Brill tagger on the corpus *train_sents*,
producing at most *max_rules* transformations, each of which
reduces the net number of errors in the corpus by at least
*min_score*, and each of which has accuracy not lower than
*min_acc*.
#imports
>>> from nltk.tbl.template import Template
>>> from nltk.tag.brill import Pos, Word
>>> from nltk.tag import untag, RegexpTagger, BrillTaggerTrainer
#some data
>>> from nltk.corpus import treebank
>>> training_data = treebank.tagged_sents()[:100]
>>> baseline_data = treebank.tagged_sents()[100:200]
>>> gold_data = treebank.tagged_sents()[200:300]
>>> testing_data = [untag(s) for s in gold_data]
>>> backoff = RegexpTagger([
... (r\'^-?[0-9]+(.[0-9]+)?$\', \'CD\'),   # cardinal numbers
... (r\'(The|the|A|a|An|an)$\', \'AT\'),   # articles
... (r\'.*able$\', \'JJ\'),                # adjectives
... (r\'.*ness$\', \'NN\'),                # nouns formed from adjectives
... (r\'.*ly$\', \'RB\'),                  # adverbs
... (r\'.*s$\', \'NNS\'),                  # plural nouns
... (r\'.*ing$\', \'VBG\'),                # gerunds
... (r\'.*ed$\', \'VBD\'),                 # past tense verbs
... (r\'.*\', \'NN\')                      # nouns (default)
>>> baseline = backoff #see NOTE1
>>> baseline.evaluate(gold_data) #doctest: +ELLIPSIS
0.2450142...
#templates
>>> Template._cleartemplates() #clear any templates created in earlier tests
>>> templates = [Template(Pos([-1])), Template(Pos([-1]), Word([0]))]
#construct a BrillTaggerTrainer
>>> tt = BrillTaggerTrainer(baseline, templates, trace=3)
>>> tagger1 = tt.train(training_data, max_rules=10)
TBL train (fast) (seqs: 100; tokens: 2417; tpls: 2; min score: 2; min acc: None)
Finding initial useful rules...
Found 845 useful rules.
<BLANKLINE>
B      |
S   F   r   O  |        Score = Fixed - Broken
c   i   o   t  |  R     Fixed = num tags changed incorrect -> correct
o   x   k   h  |  u     Broken = num tags changed correct -> incorrect
r   e   e   e  |  l     Other = num tags changed incorrect -> incorrect
e   d   n   r  |  e
132 132   0   0  | AT->DT if Pos:NN@[-1]
85  85   0   0  | NN->, if Pos:NN@[-1] & Word:,@[0]
69  69   0   0  | NN->. if Pos:NN@[-1] & Word:.@[0]
51  51   0   0  | NN->IN if Pos:NN@[-1] & Word:of@[0]
47  63  16 161  | NN->IN if Pos:NNS@[-1]
33  33   0   0  | NN->TO if Pos:NN@[-1] & Word:to@[0]
26  26   0   0  | IN->. if Pos:NNS@[-1] & Word:.@[0]
24  24   0   0  | IN->, if Pos:NNS@[-1] & Word:,@[0]
22  27   5  24  | NN->-NONE- if Pos:VBD@[-1]
17  17   0   0  | NN->CC if Pos:NN@[-1] & Word:and@[0]
>>> tagger1.rules()[1:3]
(Rule(\'001\', \'NN\', \',\', [(Pos([-1]),\'NN\'), (Word([0]),\',\')]), Rule(\'001\', \'NN\', \'.\', [(Pos([-1]),\'NN\'), (Word([0]),\'.\')]))
>>> train_stats = tagger1.train_stats()
>>> [train_stats[stat] for stat in [\'initialerrors\', \'finalerrors\', \'rulescores\']]
[1775, 1269, [132, 85, 69, 51, 47, 33, 26, 24, 22, 17]]
>>> tagger1.print_template_statistics(printunused=False)
TEMPLATE STATISTICS (TRAIN)  2 templates, 10 rules)
TRAIN (   2417 tokens) initial  1775 0.2656 final:  1269 0.4750
#ID | Score (train) |  #Rules     | Template
001 |   305   0.603 |   7   0.700 | Template(Pos([-1]),Word([0]))
000 |   201   0.397 |   3   0.300 | Template(Pos([-1]))
<BLANKLINE>
<BLANKLINE>
>>> tagger1.evaluate(gold_data) # doctest: +ELLIPSIS
0.43996...
>>> tagged, test_stats = tagger1.batch_tag_incremental(testing_data, gold_data)
>>> tagged[33][12:] == [(\'foreign\', \'IN\'), (\'debt\', \'NN\'), (\'of\', \'IN\'), (\'$\', \'NN\'), (\'64\', \'CD\'),
... (\'billion\', \'NN\'), (\'*U*\', \'NN\'), (\'--\', \'NN\'), (\'the\', \'DT\'), (\'third-highest\', \'NN\'), (\'in\', \'NN\'),
... (\'the\', \'DT\'), (\'developing\', \'VBG\'), (\'world\', \'NN\'), (\'.\', \'.\')]
True
>>> [test_stats[stat] for stat in [\'initialerrors\', \'finalerrors\', \'rulescores\']]
[1855, 1376, [100, 85, 67, 58, 27, 36, 27, 16, 31, 32]]
# a high-accuracy tagger
>>> tagger2 = tt.train(training_data, max_rules=10, min_acc=0.99)
TBL train (fast) (seqs: 100; tokens: 2417; tpls: 2; min score: 2; min acc: 0.99)
Finding initial useful rules...
Found 845 useful rules.
<BLANKLINE>
B      |
S   F   r   O  |        Score = Fixed - Broken
c   i   o   t  |  R     Fixed = num tags changed incorrect -> correct
o   x   k   h  |  u     Broken = num tags changed correct -> incorrect
r   e   e   e  |  l     Other = num tags changed incorrect -> incorrect
e   d   n   r  |  e
132 132   0   0  | AT->DT if Pos:NN@[-1]
85  85   0   0  | NN->, if Pos:NN@[-1] & Word:,@[0]
69  69   0   0  | NN->. if Pos:NN@[-1] & Word:.@[0]
51  51   0   0  | NN->IN if Pos:NN@[-1] & Word:of@[0]
36  36   0   0  | NN->TO if Pos:NN@[-1] & Word:to@[0]
26  26   0   0  | NN->. if Pos:NNS@[-1] & Word:.@[0]
24  24   0   0  | NN->, if Pos:NNS@[-1] & Word:,@[0]
19  19   0   6  | NN->VB if Pos:TO@[-1]
18  18   0   0  | CD->-NONE- if Pos:NN@[-1] & Word:0@[0]
18  18   0   0  | NN->CC if Pos:NN@[-1] & Word:and@[0]
>>> tagger2.evaluate(gold_data)  # doctest: +ELLIPSIS
0.44159544...
>>> tagger2.rules()[2:4]
(Rule(\'001\', \'NN\', \'.\', [(Pos([-1]),\'NN\'), (Word([0]),\'.\')]), Rule(\'001\', \'NN\', \'IN\', [(Pos([-1]),\'NN\'), (Word([0]),\'of\')]))
# NOTE1: (!!FIXME) A far better baseline uses nltk.tag.UnigramTagger,
# with a RegexpTagger only as backoff. For instance,
# >>> baseline = UnigramTagger(baseline_data, backoff=backoff)
# However, as of Nov 2013, nltk.tag.UnigramTagger does not yield consistent results
# between python versions. The simplistic backoff above is a workaround to make doctests
# get consistent input.
:param train_sents: training data
:type train_sents: list(list(tuple))
:param max_rules: output at most max_rules rules
:type max_rules: int
:param min_score: stop training when no rules better than min_score can be found
:type min_score: int
:param min_acc: discard any rule with lower accuracy than min_acc
:type min_acc: float or None
:return: the learned tagger
:rtype: BrillTagger'

def train(self, train_sents, max_rules=200, min_score=2, min_acc=None):

    test_sents = [list(self._initial_tagger.tag(untag(sent))) for sent in train_sents]
    trainstats = {}
    trainstats['min_acc'] = min_acc
    trainstats['min_score'] = min_score
    trainstats['tokencount'] = sum((len(t) for t in test_sents))
    trainstats['sequencecount'] = len(test_sents)
    trainstats['templatecount'] = len(self._templates)
    trainstats['rulescores'] = []
    trainstats['initialerrors'] = sum(((tag[1] != truth[1]) for paired in zip(test_sents, train_sents) for (tag, truth) in zip(*paired)))
    trainstats['initialacc'] = (1 - (trainstats['initialerrors'] / trainstats['tokencount']))
    if (self._trace > 0):
        print('TBL    train    (fast)    (seqs:    {sequencecount};    tokens:    {tokencount};    tpls:    {templatecount};    min    score:    {min_score};    min    acc:    {min_acc})'.format(**trainstats))
    if self._trace:
        print('Finding    initial    useful    rules...')
    self._init_mappings(test_sents, train_sents)
    if self._trace:
        print(('                Found    %d    useful    rules.' % len(self._rule_scores)))
    if (self._trace > 2):
        self._trace_header()
    elif (self._trace == 1):
        print('Selecting    rules...')
    rules = []
    try:
        while (len(rules) < max_rules):
            rule = self._best_rule(train_sents, test_sents, min_score, min_acc)
            if rule:
                rules.append(rule)
                score = self._rule_scores[rule]
                trainstats['rulescores'].append(score)
            else:
                break
            if (self._trace > 1):
                self._trace_rule(rule)
            self._apply_rule(rule, test_sents)
            self._update_tag_positions(rule)
            self._update_rules(rule, train_sents, test_sents)
    except KeyboardInterrupt:
        print(('Training    stopped    manually    --    %d    rules    found' % len(rules)))
    self._clean()
    trainstats['finalerrors'] = (trainstats['initialerrors'] - sum(trainstats['rulescores']))
    trainstats['finalacc'] = (1 - (trainstats['finalerrors'] / trainstats['tokencount']))
    return BrillTagger(self._initial_tagger, rules, trainstats)
