'To classify from the content we need to separate text from
binary, and figure out the encoding. This is an imperfect task.
The algorithm here is to go through the following heroics to attempt
to determine an encoding that works to decode the content. If all
such attempts fail, we presume it is binary.
1. Use the BOM, if it has one.
2. Try the given suggested encoding (if any).
3. Check for EBCDIC encoding.
4. Lang-specific (if we know the lang already):
* if this is Python, look for coding: decl and try that
* if this is Perl, look for use encoding decl and try that
5. XML: According to the XML spec the rule is the XML prolog
specifies the encoding, or it is UTF-8.
6. HTML: Attempt to use Content-Type meta tag. Try the given
charset, if any.
7. Emacs-style "coding" local var.
8. Vi[m]-style "fileencoding" local var.
9. Heuristic checks for UTF-16 without BOM.
10. Give UTF-8 a try, it is a pretty common fallback.
We must do this before a possible 8-bit
`locale.getpreferredencoding()` because any UTF-8 encoded
document will decode with an 8-bit encoding (i.e. will decode,
just with bogus characters).
11. Lang-specific fallback. E.g., UTF-8 for XML, ascii for Python.
12. chardet (http://chardet.feedparser.org/), if CHARDET_ENABLED == True
13. locale.getpreferredencoding()
14. iso8859-1 (in case `locale.getpreferredencoding()` is UTF-8
we must have an 8-bit encoding attempt).
TODO: Is there a worry for a lot of false-positives for
binary files.
Notes:
- A la Universal Feed Parser, if some
supposed-to-be-authoritative encoding indicator is wrong (e.g.
the BOM, the Python \'coding:\' decl for Python),
`self.encoding_bozo` is set True and a reason is appended to
the `self.encoding_bozo_reasons` list.'

def _classify_encoding(self, lidb, suggested_encoding=None):

    if (self.has_bom is not False):
        (self.has_bom, bom, bom_encoding) = self._get_bom_info()
        if self.has_bom:
            self._accessor.strip_bom(bom)
            if ((bom_encoding in ('utf-32-le', 'utf-32-be')) or self._accessor.decode(bom_encoding)):
                log.debug('encoding:    encoding    from    BOM:    %r', bom_encoding)
                self.encoding = bom_encoding
                return
            else:
                log.debug('encoding:    BOM    encoding    (%r)    was    *wrong*', bom_encoding)
                self._encoding_bozo((u'BOM    encoding    (%s)    could    not    decode    %s' % (bom_encoding, self._accessor)))
    head_bytes = self._accessor.head_bytes
    if DEBUG_CHARDET_INFO:
        sys.path.insert(0, os.path.expanduser('~/tm/check/contrib/chardet'))
        import chardet
        del sys.path[0]
        self.chardet_info = chardet.detect(head_bytes)
    if (suggested_encoding is not None):
        norm_suggested_encoding = _norm_encoding(suggested_encoding)
        if self._accessor.decode(suggested_encoding):
            self.encoding = norm_suggested_encoding
            return
        else:
            log.debug("encoding:    suggested    %r    encoding    didn't    work    for    %s", suggested_encoding, self._accessor)
    EBCDIC_MAGIC = 'Lo\xa7\x94'
    if (self._accessor.head_4_bytes == EBCDIC_MAGIC):
        norm_ebcdic_encoding = _norm_encoding('ebcdic-cp-us')
        if self._accessor.decode(norm_ebcdic_encoding):
            log.debug('EBCDIC    encoding:    %r', norm_ebcdic_encoding)
            self.encoding = norm_ebcdic_encoding
            return
        else:
            log.debug("EBCDIC    encoding    didn't    work    for    %s", self._accessor)
    if (self.langinfo and self.langinfo.conformant_attr('encoding_decl_pattern')):
        m = self.langinfo.conformant_attr('encoding_decl_pattern').search(head_bytes)
        if m:
            lang_encoding = m.group('encoding')
            norm_lang_encoding = _norm_encoding(lang_encoding)
            if self._accessor.decode(norm_lang_encoding):
                log.debug('encoding:    encoding    from    lang-spec:    %r', norm_lang_encoding)
                self.encoding = norm_lang_encoding
                return
            else:
                log.debug('encoding:    lang-spec    encoding    (%r)    was    *wrong*', lang_encoding)
                self._encoding_bozo((u'lang-spec    encoding    (%s)    could    not    decode    %s' % (lang_encoding, self._accessor)))
    if (self.langinfo and self.langinfo.conforms_to('XML')):
        (has_xml_prolog, xml_version, xml_encoding) = self._get_xml_prolog_info(head_bytes)
        if (xml_encoding is not None):
            norm_xml_encoding = _norm_encoding(xml_encoding)
            if self._accessor.decode(norm_xml_encoding):
                log.debug('encoding:    encoding    from    XML    prolog:    %r', norm_xml_encoding)
                self.encoding = norm_xml_encoding
                return
            else:
                log.debug('encoding:    XML    prolog    encoding    (%r)    was    *wrong*', norm_xml_encoding)
                self._encoding_bozo((u'XML    prolog    encoding    (%s)    could    not    decode    %s' % (norm_xml_encoding, self._accessor)))
    if (self.langinfo and self.langinfo.conforms_to('HTML')):
        (has_http_content_type_info, http_content_type, http_encoding) = self._get_http_content_type_info(head_bytes)
        if (has_http_content_type_info and http_encoding):
            norm_http_encoding = _norm_encoding(http_encoding)
            if self._accessor.decode(norm_http_encoding):
                log.debug('encoding:    encoding    from    HTTP    content-type:    %r', norm_http_encoding)
                self.encoding = norm_http_encoding
                return
            else:
                log.debug('encoding:    HTTP    content-type    encoding    (%r)    was    *wrong*', norm_http_encoding)
                self._encoding_bozo((u'HTML    content-type    encoding    (%s)    could    not    decode    %s' % (norm_http_encoding, self._accessor)))
    emacs_head_vars = self._get_emacs_head_vars(head_bytes)
    emacs_encoding = emacs_head_vars.get('coding')
    if (not emacs_encoding):
        tail_bytes = self._accessor.tail_bytes
        emacs_tail_vars = self._get_emacs_tail_vars(tail_bytes)
        emacs_encoding = emacs_tail_vars.get('coding')
    if emacs_encoding:
        norm_emacs_encoding = _norm_encoding(emacs_encoding)
        if self._accessor.decode(norm_emacs_encoding):
            log.debug('encoding:    encoding    from    Emacs    coding    var:    %r', norm_emacs_encoding)
            self.encoding = norm_emacs_encoding
            return
        else:
            log.debug('encoding:    Emacs    coding    var    (%r)    was    *wrong*', norm_emacs_encoding)
            self._encoding_bozo((u'Emacs    coding    var    (%s)    could    not    decode    %s' % (norm_emacs_encoding, self._accessor)))
    vi_vars = self._get_vi_vars(head_bytes)
    vi_encoding = (vi_vars.get('fileencoding') or vi_vars.get('fenc'))
    if (not vi_encoding):
        vi_vars = self._get_vi_vars(self._accessor.tail_bytes)
        vi_encoding = (vi_vars.get('fileencoding') or vi_vars.get('fenc'))
    if vi_encoding:
        norm_vi_encoding = _norm_encoding(vi_encoding)
        if self._accessor.decode(norm_vi_encoding):
            log.debug('encoding:    encoding    from    Vi[m]    coding    var:    %r', norm_vi_encoding)
            self.encoding = norm_vi_encoding
            return
        else:
            log.debug('encoding:    Vi[m]    coding    var    (%r)    was    *wrong*', norm_vi_encoding)
            self._encoding_bozo((u'Vi[m]    coding    var    (%s)    could    not    decode    %s' % (norm_vi_encoding, self._accessor)))
    utf16_encoding = None
    head_odd_bytes = head_bytes[0::2]
    head_even_bytes = head_bytes[1::2]
    head_markers = ['<?xml', '#!']
    for head_marker in head_markers:
        length = len(head_marker)
        if (head_odd_bytes.startswith(head_marker) and (head_even_bytes[0:length] == ('\x00' * length))):
            utf16_encoding = 'utf-16-le'
            break
        elif (head_even_bytes.startswith(head_marker) and (head_odd_bytes[0:length] == ('\x00' * length))):
            utf16_encoding = 'utf-16-be'
            break
    internal_markers = ['coding']
    for internal_marker in internal_markers:
        length = len(internal_marker)
        try:
            idx = head_odd_bytes.index(internal_marker)
        except ValueError:
            pass
        else:
            if (head_even_bytes[idx:(idx + length)] == ('\x00' * length)):
                utf16_encoding = 'utf-16-le'
        try:
            idx = head_even_bytes.index(internal_marker)
        except ValueError:
            pass
        else:
            if (head_odd_bytes[idx:(idx + length)] == ('\x00' * length)):
                utf16_encoding = 'utf-16-be'
    if utf16_encoding:
        if self._accessor.decode(utf16_encoding):
            log.debug('encoding:    guessed    encoding:    %r', utf16_encoding)
            self.encoding = utf16_encoding
            return
    norm_utf8_encoding = _norm_encoding('utf-8')
    if self._accessor.decode(norm_utf8_encoding):
        log.debug('UTF-8    encoding:    %r', norm_utf8_encoding)
        self.encoding = norm_utf8_encoding
        return
    fallback_encoding = None
    fallback_lang = None
    if self.langinfo:
        fallback_lang = self.langinfo.name
        fallback_encoding = self.langinfo.conformant_attr('default_encoding')
    if fallback_encoding:
        if self._accessor.decode(fallback_encoding):
            log.debug('encoding:    fallback    encoding    for    %s:    %r', fallback_lang, fallback_encoding)
            self.encoding = fallback_encoding
            return
        else:
            log.debug('encoding:    %s    fallback    encoding    (%r)    was    *wrong*', fallback_lang, fallback_encoding)
            self._encoding_bozo((u'%s    fallback    encoding    (%s)    could    not    decode    %s' % (fallback_lang, fallback_encoding, self._accessor)))
    if (self.CHARDET_ENABLED and self.langinfo and self.langinfo.is_text):
        try:
            import chardet
        except ImportError:
            warnings.warn('no    chardet    module    to    aid    in    guessing    encoding', ChardetImportWarning)
        else:
            chardet_info = chardet.detect(head_bytes)
            if (chardet_info['encoding'] and (chardet_info['confidence'] > self.CHARDET_THRESHHOLD)):
                chardet_encoding = chardet_info['encoding']
                norm_chardet_encoding = _norm_encoding(chardet_encoding)
                if self._accessor.decode(norm_chardet_encoding):
                    log.debug('chardet    encoding:    %r', chardet_encoding)
                    self.encoding = norm_chardet_encoding
                    return
    locale_encoding = locale.getpreferredencoding()
    if locale_encoding:
        norm_locale_encoding = _norm_encoding(locale_encoding)
        if self._accessor.decode(norm_locale_encoding):
            log.debug('encoding:    locale    preferred    encoding:    %r', locale_encoding)
            self.encoding = norm_locale_encoding
            return
    norm_fallback8bit_encoding = _norm_encoding('iso8859-1')
    if self._accessor.decode(norm_fallback8bit_encoding):
        log.debug('fallback    8-bit    encoding:    %r', norm_fallback8bit_encoding)
        self.encoding = norm_fallback8bit_encoding
        return
    self.is_text = False
