DataUtil { static Document parseByteData(ByteBuffer byteData, String charsetName, String baseUri, Parser parser) { String docData; Document doc = null; charsetName = detectCharsetFromBom(byteData, charsetName); if (charsetName == null) { docData = Charset.forName(defaultCharset).decode(byteData).toString(); doc = parser.parseInput(docData, baseUri); Element meta = doc.select("meta[http-equiv=content-type], meta[charset]").first(); String foundCharset = null; if (meta != null) { if (meta.hasAttr("http-equiv")) { foundCharset = getCharsetFromContentType(meta.attr("content")); } if (foundCharset == null && meta.hasAttr("charset")) { foundCharset = meta.attr("charset"); } } if (foundCharset == null && doc.childNode(0) instanceof XmlDeclaration) { XmlDeclaration prolog = (XmlDeclaration) doc.childNode(0); if (prolog.name().equals("xml")) { foundCharset = prolog.attr("encoding"); } } foundCharset = validateCharset(foundCharset); if (foundCharset != null && !foundCharset.equals(defaultCharset)) { foundCharset = foundCharset.trim().replaceAll("[\"']", ""); charsetName = foundCharset; ((Buffer) byteData).rewind(); docData = Charset.forName(foundCharset).decode(byteData).toString(); doc = null; } } else { Validate.notEmpty(charsetName, "Must set charset arg to character set of file to parse. Set to null to attempt to detect from HTML"); docData = Charset.forName(charsetName).decode(byteData).toString(); } if (doc == null) { doc = parser.parseInput(docData, baseUri); doc.outputSettings().charset(charsetName); } return doc; } private  DataUtil(); static Document load(File in, String charsetName, String baseUri); static Document load(InputStream in, String charsetName, String baseUri); static Document load(InputStream in, String charsetName, String baseUri, Parser parser);  }
@Test public void discardsSpuriousByteOrderMark() { String html = "\uFEFF<html><head><title>One</title></head><body>Two</body></html>"; ByteBuffer buffer = Charset.forName("UTF-8").encode(html); Document doc = DataUtil.parseByteData(buffer, "UTF-8", "http: assertEquals("One", doc.head().text()); }
@Test public void discardsSpuriousByteOrderMarkWhenNoCharsetSet() { String html = "\uFEFF<html><head><title>One</title></head><body>Two</body></html>"; ByteBuffer buffer = Charset.forName("UTF-8").encode(html); Document doc = DataUtil.parseByteData(buffer, null, "http: assertEquals("One", doc.head().text()); assertEquals("UTF-8", doc.outputSettings().charset().displayName()); }
@Test public void wrongMetaCharsetFallback() { try { final byte[] input = "<html><head><meta charset=iso-8></head><body></body></html>".getBytes("UTF-8"); final ByteBuffer inBuffer = ByteBuffer.wrap(input); Document doc = DataUtil.parseByteData(inBuffer, null, "http: final String expected = "<html>\n" + " <head>\n" + " <meta charset=\"iso-8\">\n" + " </head>\n" + " <body></body>\n" + "</html>"; assertEquals(expected, doc.toString()); } catch( UnsupportedEncodingException ex ) { fail(ex.getMessage()); } }