GuessedEncoding (int startOffset, int endOffset) { // if a byte has its most significant bit set, the file is in UTF-8 or in the default encoding // otherwise, the file is in US-ASCII boolean highOrderBit = false; // if the file is in UTF-8, high order bytes must have a certain value, in order to be valid // if it's not the case, we can assume the encoding is the default encoding of the system boolean validU8Char = true; // true if char bytes < BINARY_THRESHOLD occurred boolean hasBinary = false; int end = Math.min(buffer.length, endOffset); int i = startOffset; while (i < end) { byte b0 = buffer[i]; byte b1 = i + 1 >= end ? 0 : buffer[i + 1]; byte b2 = i + 2 >= end ? 0 : buffer[i + 2]; byte b3 = i + 3 >= end ? 0 : buffer[i + 3]; byte b4 = i + 4 >= end ? 0 : buffer[i + 4]; byte b5 = i + 5 >= end ? 0 : buffer[i + 5]; if (b0 < 0) { // a high order bit was encountered, thus the encoding is not US-ASCII // it may be either an 8-bit encoding or UTF-8 highOrderBit = true; // a two-bytes sequence was encountered if (isTwoBytesSequence(b0)) { // there must be one continuation byte of the form 10xxxxxx, // otherwise the following characters is not a valid UTF-8 construct if (!isContinuationChar(b1)) { validU8Char = false; } else { i++; } } // a three-bytes sequence was encountered else if (isThreeBytesSequence(b0)) { // there must be two continuation bytes of the form 10xxxxxx, // otherwise the following characters is not a valid UTF-8 construct if (!(isContinuationChar(b1) && isContinuationChar(b2))) { validU8Char = false; } else { i += 2; } } // a four-bytes sequence was encountered else if (isFourBytesSequence(b0)) { // there must be three continuation bytes of the form 10xxxxxx, // otherwise the following characters is not a valid UTF-8 construct if (!(isContinuationChar(b1) && isContinuationChar(b2) && isContinuationChar(b3))) { validU8Char = false; } else { i += 3; } } // a five-bytes sequence was encountered else if (isFiveBytesSequence(b0)) { // there must be four continuation bytes of the form 10xxxxxx, // otherwise the following characters is not a valid UTF-8 construct if (!(isContinuationChar(b1) && isContinuationChar(b2) && isContinuationChar(b3) && isContinuationChar(b4))) { validU8Char = false; } else { i += 4; } } // a six-bytes sequence was encountered else if (isSixBytesSequence(b0)) { // there must be five continuation bytes of the form 10xxxxxx, // otherwise the following characters is not a valid UTF-8 construct if (!(isContinuationChar(b1) && isContinuationChar(b2) && isContinuationChar(b3) && isContinuationChar(b4) && isContinuationChar(b5))) { validU8Char = false; } else { i += 5; } } else { validU8Char = false; } } else if (b0 < BINARY_THRESHOLD) { hasBinary = true; break; } if (!validU8Char) break; i++; } if (!highOrderBit && !hasBinary) { return GuessedEncoding.SEVEN_BIT; } // finally, if it's not UTF-8 nor US-ASCII if (!validU8Char) return GuessedEncoding.INVALID_UTF8; if (hasBinary) return GuessedEncoding.BINARY; // if no invalid UTF-8 were encountered, we can assume the encoding is UTF-8, // otherwise the file would not be human readable return GuessedEncoding.VALID_UTF8; }