IOUtils.write() throws
OutOfMemoryError/NegativeArraySizeException while writing big strings
Adapt PR by DaGeRe.
Use a ternary expression.
MODIFY
src/changes/changes.xml
src/changes/changes.xml
@@ -165,6 +165,9 @@ The <action> type attribute can be add,update,fix,remove.
<action dev="ggregory" type="fix" due-to="richarda23">
FileFilterTest minor fixes #340.
</action>
+ <action issue="IO-764" dev="ggregory" type="fix" due-to="DaGeRe, Gary Gregory">
+ IOUtils.write() throws OutOfMemoryError/NegativeArraySizeException while writing big strings #343.
+ </action>
<!-- ADD -->
<action issue="IO-726" dev="ggregory" type="fix" due-to="shollander, Gary Gregory">
Add MemoryMappedFileInputStream #215.

MODIFY
src/main/java/org/apache/commons/io/IOUtils.java
src/main/java/org/apache/commons/io/IOUtils.java
import java.net.URLConnection;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
+import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.Selector;
import java.nio.charset.Charset;
@@ -968,10 +969,7 @@ public static boolean contentEqualsIgnoreEOL(final Reader reader1, final Reader
*/
public static int copy(final InputStream inputStream, final OutputStream outputStream) throws IOException {
final long count = copyLarge(inputStream, outputStream);
- if (count > Integer.MAX_VALUE) {
- return EOF;
- }
- return (int) count;
+ return count > Integer.MAX_VALUE ? EOF : (int) count;
}
/**
@@ -3360,9 +3358,13 @@ public static void write(final String data, final OutputStream output)
* @throws IOException if an I/O error occurs
* @since 2.3
*/
+ @SuppressWarnings("resource")
public static void write(final String data, final OutputStream output, final Charset charset) throws IOException {
if (data != null) {
- output.write(data.getBytes(Charsets.toCharset(charset)));
+ // Use Charset#encode(String), since calling String#getBytes(Charset) might result in
+ // NegativeArraySizeException or OutOfMemoryError.
+ // The underlying OutputStream should not be closed, so the channel is not closed.
+ Channels.newChannel(output).write(Charsets.toCharset(charset).encode(data));
}
}

MODIFY
src/test/java/org/apache/commons/io/IOUtilsTest.java
src/test/java/org/apache/commons/io/IOUtilsTest.java
@@ -292,39 +292,6 @@ public void testAsWriterStringBuilder() throws IOException {
}
}
- /**
- * IO-764 IOUtils.write() throws NegativeArraySizeException while writing big strings.
- * <pre>
- * java.lang.OutOfMemoryError: Java heap space
- * at java.lang.StringCoding.encode(StringCoding.java:350)
- * at java.lang.String.getBytes(String.java:941)
- * at org.apache.commons.io.IOUtils.write(IOUtils.java:3367)
- * at org.apache.commons.io.IOUtilsTest.testBigString(IOUtilsTest.java:1659)
- * </pre>
- */
- @Test
- public void testBigString() throws IOException {
- // 3_000_000 is a size that we can allocate for the test string with Java 8 on the command line as:
- // mvn clean test -Dtest=IOUtilsTest -DtestBigString=3000000
- // 6_000_000 failed with the above
- //
- // TODO Can we mock the test string for this test to pretend to be larger?
- // Mocking the length seems simple but how about the data?
- final int repeat = Integer.getInteger("testBigString", 3_000_000);
- final String data;
- try {
- data = StringUtils.repeat("\uD83D", repeat);
- } catch (OutOfMemoryError e) {
- System.err.printf("Don't fail the test if we cannot build the fixture, just log, fixture size = %,d%n.", repeat);
- e.printStackTrace();
- return;
- }
- try (CountingOutputStream os = new CountingOutputStream(NullOutputStream.INSTANCE)) {
- IOUtils.write(data, os, StandardCharsets.UTF_8);
- assertEquals(repeat, os.getByteCount());
- }
- }
-
@Test
public void testClose() {
assertDoesNotThrow(() -> IOUtils.close((Closeable) null));
@@ -1673,4 +1640,49 @@ public void testToString_URL_CharsetNameNull() throws Exception {
testToString_URL(null);
}
+ /**
+ * IO-764 IOUtils.write() throws NegativeArraySizeException while writing big strings.
+ * <pre>
+ * java.lang.OutOfMemoryError: Java heap space
+ * at java.lang.StringCoding.encode(StringCoding.java:350)
+ * at java.lang.String.getBytes(String.java:941)
+ * at org.apache.commons.io.IOUtils.write(IOUtils.java:3367)
+ * at org.apache.commons.io.IOUtilsTest.testBigString(IOUtilsTest.java:1659)
+ * </pre>
+ */
+ @Test
+ public void testWriteBigString() throws IOException {
+ // 3_000_000 is a size that we can allocate for the test string with Java 8 on the command line as:
+ // mvn clean test -Dtest=IOUtilsTest -DtestBigString=3000000
+ // 6_000_000 failed with the above
+ //
+ // TODO Can we mock the test string for this test to pretend to be larger?
+ // Mocking the length seems simple but how about the data?
+ final int repeat = Integer.getInteger("testBigString", 3_000_000);
+ final String data;
+ try {
+ data = StringUtils.repeat("\uD83D", repeat);
+ } catch (OutOfMemoryError e) {
+ System.err.printf("Don't fail the test if we cannot build the fixture, just log, fixture size = %,d%n.", repeat);
+ e.printStackTrace();
+ return;
+ }
+ try (CountingOutputStream os = new CountingOutputStream(NullOutputStream.INSTANCE)) {
+ IOUtils.write(data, os, StandardCharsets.UTF_8);
+ assertEquals(repeat, os.getByteCount());
+ }
+ }
+
+ @Test
+ public void testWriteLittleString() throws IOException {
+ final String data = "\uD83D";
+ // White-box test to check that not closing the internal channel is not a problem.
+ for (int i = 0; i < 1_000_000; i++) {
+ try (CountingOutputStream os = new CountingOutputStream(NullOutputStream.INSTANCE)) {
+ IOUtils.write(data, os, StandardCharsets.UTF_8);
+ assertEquals(data.length(), os.getByteCount());
+ }
+ }
+ }
+
}

[IO-764] IOUtils.write() throws
OutOfMemoryError/NegativeArraySizeException while writing big strings

- Adapt PR #343 by DaGeRe.
- Use a ternary expression.