'Test the break iterator'

def test_break_iterator(self):

    from calibre.spell.break_iterator import split_into_words as split, index_of, split_into_words_and_positions
    for q in (u'one    two    three', u'    one    two    three', u'one\ntwo        three    '):
        self.ae(split(unicode(q)), [u'one', u'two', u'three'], (u'Failed    to    split:    %r' % q))
    self.ae(split(u"I    I'm"), [u'I', u"I'm"])
    self.ae(split(u'out-of-the-box'), [u'out-of-the-box'])
    self.ae(split(u'-one    two-'), [u'-one', u'two-'])
    self.ae(split(u'-one    a-b-c-d    e'), [u'-one', u'a-b-c-d', u'e'])
    self.ae(split(u'-one    -a-b-c-d-    e'), [u'-one', u'-a-b-c-d-', u'e'])
    self.ae(split_into_words_and_positions(u'one    \U0001f431    three'), [(0, 3), ((7 if icu.is_narrow_build else 6), 5)])
    for (needle, haystack, pos) in ((u'word', u'a    word    b', 2), (u'word', u'a    word', 2), (u'one-two', u'a    one-two    punch', 2), (u'one-two', u'one-two    punch', 0), (u'one-two', u'one-two', 0), (u'one', u'one-two    one', 8), (u'one-two', u'one-two-three    one-two', 14), (u'one', u'onet    one', 5), (u'two', u'one-two    two', 8), (u'two', u'two-one    two', 8), (u'-two', u'one-two    -two', 8), (u'-two', u'two', (-1)), (u'i', u'i', 0), (u'i', u'six    i', 4), (u'i', u'', (-1)), (u'', u'', (-1)), (u'', u'i', (-1)), (u'i', u'six    clicks', (-1)), (u'i', u'\U0001f431    i', (3 if icu.is_narrow_build else 2)), (u'-a', u'b    -a', 2), (u'a-', u'a-b    a-    d', 4), (u'-a-', u'b    -a    -a-', 5), (u'-a-', u'-a-', 0), (u'-a-', u'a-', (-1)), (u'-a-', u'-a', (-1)), (u'-a-', u'a', (-1)), (u'a-', u'a-', 0), (u'-a', u'-a', 0), (u'a-b-c-', u'a-b-c-d', (-1)), (u'a-b-c-', u'a-b-c-.', 0), (u'a-b-c-', u'a-b-c-d    a-b-c-    d', 8)):
        fpos = index_of(needle, haystack)
        self.ae(pos, fpos, (u'Failed    to    find    index    of    %r    in    %r    (%d    !=    %d)' % (needle, haystack, pos, fpos)))
