'Take a title and return a list of tokens useful for an AND search query.
Excludes connectives(optionally) and punctuation.'

def get_title_tokens(self, title, strip_joiners=True, strip_subtitle=False):

    if title:
        if strip_subtitle:
            subtitle = re.compile(u'([\\(\\[\\{].*?[\\)\\]\\}]|[/:\\\\].*$)')
            if (len(subtitle.sub(u'', title)) > 1):
                title = subtitle.sub(u'', title)
        title_patterns = [(re.compile(pat, re.IGNORECASE), repl) for (pat, repl) in [(u'(?i)[({\\[](\\d{4}|omnibus|anthology|hardcover|audiobook|audio\\scd|paperback|turtleback|mass\\s*market|edition|ed\\.)[\\])}]', u''), (u'(?i)[({\\[].*?(edition|ed.).*?[\\]})]', u''), (u'(\\d+),(\\d+)', u'\\1\\2'), (u'(\\s-)', u'    '), (u'[:,;!@$%^&*(){}.`~"\\s\\[\\]/]', u'    ')]]
        for (pat, repl) in title_patterns:
            title = pat.sub(repl, title)
        tokens = title.split()
        for token in tokens:
            token = token.strip().strip(u'"').strip(u"'")
            if (token and ((not strip_joiners) or (token.lower() not in (u'a', u'and', u'the', u'&')))):
                (yield token)
