alluxio-master-2.x/tests/src/test/java/alluxio/client/hadoop/HdfsFileInputStreamIntegrationTest.java
/*
 * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
 * (the "License"). You may not use this work except in compliance with the License, which is
 * available at www.apache.org/licenses/LICENSE-2.0
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied, as more fully set forth in the License.
 *
 * See the NOTICE file distributed with this work for information regarding copyright ownership.
 */

package alluxio.client.hadoop;

import alluxio.AlluxioURI;
import alluxio.client.ReadType;
import alluxio.client.file.FileSystem;
import alluxio.client.file.FileSystemTestUtils;
import alluxio.client.file.FileSystemUtils;
import alluxio.client.file.URIStatus;
import alluxio.conf.Configuration;
import alluxio.conf.InstancedConfiguration;
import alluxio.conf.PropertyKey;
import alluxio.exception.AlluxioException;
import alluxio.exception.PreconditionMessage;
import alluxio.grpc.WritePType;
import alluxio.hadoop.HadoopClientTestUtils;
import alluxio.hadoop.HdfsFileInputStream;
import alluxio.testutils.BaseIntegrationTest;
import alluxio.testutils.LocalAlluxioClusterResource;
import alluxio.util.io.BufferUtils;

import org.apache.hadoop.fs.Seekable;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;

import java.io.EOFException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Arrays;

/**
 * Integration tests for {@link HdfsFileInputStream}.
 */
public final class HdfsFileInputStreamIntegrationTest extends BaseIntegrationTest {

  private static final int FILE_LEN = 255;
  private static final int BUFFER_SIZE = 50;
  private static final String IN_MEMORY_FILE = "/inMemoryFile";
  private static final String UFS_ONLY_FILE = "/ufsOnlyFile";

  @ClassRule
  public static LocalAlluxioClusterResource sLocalAlluxioClusterResource =
      new LocalAlluxioClusterResource.Builder().build();
  private FileSystem mFileSystem;
  private HdfsFileInputStream mInMemInputStream;
  private HdfsFileInputStream mUfsInputStream;

  @Rule
  public final ExpectedException mThrown = ExpectedException.none();

  @After
  public void after() throws IOException, AlluxioException {
    mInMemInputStream.close();
    mFileSystem.delete(new AlluxioURI(IN_MEMORY_FILE));
    if (mUfsInputStream != null) {
      mUfsInputStream.close();
      mFileSystem.delete(new AlluxioURI(UFS_ONLY_FILE));
    }
    HadoopClientTestUtils.disableMetrics(Configuration.modifiableGlobal());
  }

  @Before
  public final void before() throws Exception {
    mFileSystem = sLocalAlluxioClusterResource.get().getClient();
    FileSystemTestUtils
        .createByteFile(mFileSystem, IN_MEMORY_FILE, WritePType.CACHE_THROUGH, FILE_LEN);
    mInMemInputStream = new HdfsFileInputStream(mFileSystem,
        new AlluxioURI(IN_MEMORY_FILE), null);
  }

  private void createUfsInStream(ReadType readType) throws Exception {
    InstancedConfiguration conf = Configuration.copyGlobal();
    conf.set(PropertyKey.USER_FILE_READ_TYPE_DEFAULT, readType.name());
    FileSystem fs = FileSystem.Factory.create(conf);
    FileSystemTestUtils.createByteFile(fs, UFS_ONLY_FILE, WritePType.THROUGH,
        FILE_LEN);
    mUfsInputStream = new HdfsFileInputStream(fs, new AlluxioURI(UFS_ONLY_FILE), null);
  }

  /**
   * Tests {@link HdfsFileInputStream#available()}.
   */
  @Test
  public void available() throws Exception {
    Assert.assertEquals(FILE_LEN, mInMemInputStream.available());
    createUfsInStream(ReadType.NO_CACHE);
    Assert.assertEquals(FILE_LEN, mUfsInputStream.available());

    // Advance the streams and check available() again.
    byte[] buf = new byte[BUFFER_SIZE];
    int length1 = mInMemInputStream.read(buf);
    Assert.assertEquals(FILE_LEN - length1, mInMemInputStream.available());
    int length2 = mUfsInputStream.read(buf);
    Assert.assertEquals(FILE_LEN - length2, mUfsInputStream.available());
  }

  /**
   * Tests {@link HdfsFileInputStream#read()}.
   */
  @Test
  public void readTest1() throws Exception {
    createUfsInStream(ReadType.NO_CACHE);
    for (int i = 0; i < FILE_LEN; i++) {
      int value = mInMemInputStream.read();
      Assert.assertEquals(i & 0x00ff, value);
      value = mUfsInputStream.read();
      Assert.assertEquals(i & 0x00ff, value);
    }
    Assert.assertEquals(FILE_LEN, mInMemInputStream.getPos());
    Assert.assertEquals(FILE_LEN, mUfsInputStream.getPos());

    int value = mInMemInputStream.read();
    Assert.assertEquals(-1, value);

    value = mUfsInputStream.read();
    Assert.assertEquals(-1, value);
  }

  /**
   * Tests {@link HdfsFileInputStream#read(byte[])}.
   */
  @Test
  public void readTest2() throws Exception {
    byte[] buf = new byte[FILE_LEN];
    int length = mInMemInputStream.read(buf);
    Assert.assertEquals(FILE_LEN, length);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(FILE_LEN, buf));

    createUfsInStream(ReadType.NO_CACHE);
    Arrays.fill(buf, (byte) 0);
    length = mUfsInputStream.read(buf);
    Assert.assertEquals(FILE_LEN, length);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(FILE_LEN, buf));

    Arrays.fill(buf, (byte) 0);
    length = mInMemInputStream.read(buf);
    Assert.assertEquals(-1, length);
    length = mUfsInputStream.read(buf);
    Assert.assertEquals(-1, length);
  }

  /**
   * Tests {@link HdfsFileInputStream#read(byte[], int, int)}.
   */
  @Test
  public void readTest3() throws Exception {
    byte[] buf = new byte[FILE_LEN];
    int length = mInMemInputStream.read(buf, 0, FILE_LEN);
    Assert.assertEquals(FILE_LEN, length);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(FILE_LEN, buf));

    createUfsInStream(ReadType.NO_CACHE);
    Arrays.fill(buf, (byte) 0);
    length = mUfsInputStream.read(buf, 0, FILE_LEN);
    Assert.assertEquals(FILE_LEN, length);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(FILE_LEN, buf));

    Arrays.fill(buf, (byte) 0);
    length = mInMemInputStream.read(buf, 0, 1);
    Assert.assertEquals(-1, length);
    length = mUfsInputStream.read(buf, 0, 1);
    Assert.assertEquals(-1, length);
  }

  /**
   * Tests {@link HdfsFileInputStream#read(long, byte[], int, int)}.
   */
  @Test
  public void readTest4() throws Exception {
    byte[] buf = new byte[FILE_LEN];
    int length = mInMemInputStream.read(0, buf, 0, FILE_LEN);
    Assert.assertEquals(FILE_LEN, length);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(FILE_LEN, buf));
    Assert.assertEquals(0, mInMemInputStream.getPos());

    createUfsInStream(ReadType.NO_CACHE);
    Arrays.fill(buf, (byte) 0);
    length = mUfsInputStream.read(0, buf, 0, FILE_LEN);
    Assert.assertEquals(FILE_LEN, length);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(FILE_LEN, buf));
    Assert.assertEquals(0, mUfsInputStream.getPos());

    buf = new byte[FILE_LEN - 10];
    Arrays.fill(buf, (byte) 0);
    length = mInMemInputStream.read(10, buf, 0, FILE_LEN - 10);
    Assert.assertEquals(FILE_LEN - 10, length);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, FILE_LEN - 10, buf));
    Assert.assertEquals(0, mInMemInputStream.getPos());

    Arrays.fill(buf, (byte) 0);
    length = mUfsInputStream.read(10, buf, 0, FILE_LEN - 10);
    Assert.assertEquals(FILE_LEN - 10, length);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, FILE_LEN - 10, buf));
    Assert.assertEquals(0, mUfsInputStream.getPos());

    Arrays.fill(buf, (byte) 0);
    length = mInMemInputStream.read(-1, buf, 0, FILE_LEN);
    Assert.assertEquals(-1, length);
    length = mUfsInputStream.read(-1, buf, 0, FILE_LEN);
    Assert.assertEquals(-1, length);

    length = mInMemInputStream.read(FILE_LEN, buf, 0, FILE_LEN);
    Assert.assertEquals(-1, length);
    length = mUfsInputStream.read(FILE_LEN, buf, 0, FILE_LEN);
    Assert.assertEquals(-1, length);
  }

  /**
   * Tests {@link HdfsFileInputStream#read(java.nio.ByteBuffer buf)}.
   */
  @Test
  public void readTest5() throws Exception {
    byte[] data = new byte[FILE_LEN];
    ByteBuffer buf = ByteBuffer.wrap(data);
    int length = mInMemInputStream.read(buf);
    Assert.assertEquals(FILE_LEN, length);
    Assert.assertTrue(BufferUtils.equalIncreasingByteBuffer(0, FILE_LEN, buf));

    createUfsInStream(ReadType.NO_CACHE);
    buf.rewind();
    buf.clear();
    length = mUfsInputStream.read(buf);
    Assert.assertEquals(FILE_LEN, length);
    Assert.assertTrue(BufferUtils.equalIncreasingByteBuffer(0, FILE_LEN, buf));

    buf.rewind();
    buf.clear();
    length = mInMemInputStream.read(buf);
    Assert.assertEquals(-1, length);
    length = mUfsInputStream.read(buf);
    Assert.assertEquals(-1, length);
  }

  /**
   * Tests {@link HdfsFileInputStream#readFully(long, byte[])}.
   */
  @Test
  public void readFullyTest1() throws Exception {
    byte[] buf = new byte[FILE_LEN];
    mInMemInputStream.readFully(0, buf);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(FILE_LEN, buf));
    Assert.assertEquals(0, mInMemInputStream.getPos());

    createUfsInStream(ReadType.NO_CACHE);
    Arrays.fill(buf, (byte) 0);
    mUfsInputStream.readFully(0, buf);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(FILE_LEN, buf));
    Assert.assertEquals(0, mUfsInputStream.getPos());

    buf = new byte[FILE_LEN - 10];
    Arrays.fill(buf, (byte) 0);
    mInMemInputStream.readFully(10, buf);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, FILE_LEN - 10, buf));
    Assert.assertEquals(0, mInMemInputStream.getPos());

    Arrays.fill(buf, (byte) 0);
    mUfsInputStream.readFully(10, buf);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, FILE_LEN - 10, buf));
    Assert.assertEquals(0, mUfsInputStream.getPos());

    Arrays.fill(buf, (byte) 0);
    try {
      mInMemInputStream.readFully(-1, buf);
      Assert.fail("readFully() is expected to fail");
    } catch (EOFException e) {
      // this is expected
    }
    BufferUtils.equalConstantByteArray((byte) 0, FILE_LEN, buf);
    try {
      mUfsInputStream.readFully(-1, buf);
      Assert.fail("readFully() is expected to fail");
    } catch (EOFException e) {
      // this is expected
    }
    BufferUtils.equalConstantByteArray((byte) 0, FILE_LEN, buf);

    try {
      mInMemInputStream.readFully(FILE_LEN, buf);
      Assert.fail("readFully() is expected to fail");
    } catch (EOFException e) {
      // this is expected
    }
    BufferUtils.equalConstantByteArray((byte) 0, FILE_LEN, buf);
    try {
      mUfsInputStream.readFully(FILE_LEN, buf);
      Assert.fail("readFully() is expected to fail");
    } catch (EOFException e) {
      // this is expected
    }
    BufferUtils.equalConstantByteArray((byte) 0, FILE_LEN, buf);
  }

  /**
   * Tests {@link HdfsFileInputStream#readFully(long, byte[], int, int)}.
   */
  @Test
  public void readFullyTest2() throws Exception {
    byte[] buf = new byte[FILE_LEN];
    mInMemInputStream.readFully(0, buf, 0, FILE_LEN);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(FILE_LEN, buf));
    Assert.assertEquals(0, mInMemInputStream.getPos());

    createUfsInStream(ReadType.NO_CACHE);
    Arrays.fill(buf, (byte) 0);
    mUfsInputStream.readFully(0, buf, 0, FILE_LEN);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(FILE_LEN, buf));
    Assert.assertEquals(0, mUfsInputStream.getPos());

    buf = new byte[FILE_LEN - 10];
    Arrays.fill(buf, (byte) 0);
    mInMemInputStream.readFully(10, buf, 0, FILE_LEN - 10);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, FILE_LEN - 10, buf));
    Assert.assertEquals(0, mInMemInputStream.getPos());

    Arrays.fill(buf, (byte) 0);
    mUfsInputStream.readFully(10, buf, 0, FILE_LEN - 10);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, FILE_LEN - 10, buf));
    Assert.assertEquals(0, mUfsInputStream.getPos());

    Arrays.fill(buf, (byte) 0);
    try {
      mInMemInputStream.readFully(-1, buf, 0, FILE_LEN);
      Assert.fail("readFully() is expected to fail");
    } catch (EOFException e) {
      // this is expected
    }
    BufferUtils.equalConstantByteArray((byte) 0, FILE_LEN, buf);
    try {
      mUfsInputStream.readFully(-1, buf, 0, FILE_LEN);
      Assert.fail("readFully() is expected to fail");
    } catch (EOFException e) {
      // this is expected
    }
    BufferUtils.equalConstantByteArray((byte) 0, FILE_LEN, buf);

    try {
      mInMemInputStream.readFully(FILE_LEN, buf, 0, FILE_LEN);
      Assert.fail("readFully() is expected to fail");
    } catch (EOFException e) {
      // this is expected
    }
    BufferUtils.equalConstantByteArray((byte) 0, FILE_LEN, buf);
    try {
      mUfsInputStream.readFully(FILE_LEN, buf, 0, FILE_LEN);
      Assert.fail("readFully() is expected to fail");
    } catch (EOFException e) {
      // this is expected
    }
    BufferUtils.equalConstantByteArray((byte) 0, FILE_LEN, buf);
  }

  @Test
  public void inMemSeek() throws Exception {
    seekTest(mInMemInputStream);
  }

  @Test
  public void ufsSeek() throws Exception {
    createUfsInStream(ReadType.NO_CACHE);
    seekTest(mUfsInputStream);
  }

  private void seekTest(Seekable stream) throws Exception {
    stream.seek(0);
    Assert.assertEquals(0, stream.getPos());

    stream.seek(FILE_LEN / 2);
    Assert.assertEquals(FILE_LEN / 2, stream.getPos());

    stream.seek(1);
    Assert.assertEquals(1, stream.getPos());
  }

  @Test
  public void seekNegative() throws Exception {
    mThrown.expect(IOException.class);
    mThrown.expectMessage(String.format(PreconditionMessage.ERR_SEEK_NEGATIVE.toString(), -1));
    mInMemInputStream.seek(-1);
  }

  @Test
  public void seekPastEof() throws Exception {
    mThrown.expect(IOException.class);
    mThrown.expectMessage(String.format(PreconditionMessage.ERR_SEEK_PAST_END_OF_FILE.toString(),
        FILE_LEN + 1));
    mInMemInputStream.seek(FILE_LEN + 1);
  }

  @Test
  public void seekNegativeUfs() throws Exception {
    mThrown.expect(IOException.class);
    mThrown.expectMessage(String.format(PreconditionMessage.ERR_SEEK_NEGATIVE.toString(), -1));
    createUfsInStream(ReadType.NO_CACHE);
    mUfsInputStream.seek(-1);
  }

  @Test
  public void seekPastEofUfs() throws Exception {
    mThrown.expect(IOException.class);
    mThrown.expectMessage(String.format(PreconditionMessage.ERR_SEEK_PAST_END_OF_FILE.toString(),
        FILE_LEN + 1));
    createUfsInStream(ReadType.NO_CACHE);
    mUfsInputStream.seek(FILE_LEN + 1);
  }

  @Test
  public void positionedReadCache() throws Exception {
    createUfsInStream(ReadType.CACHE);
    mUfsInputStream.readFully(0, new byte[FILE_LEN]);
    FileSystemUtils.waitForAlluxioPercentage(mFileSystem, new AlluxioURI(UFS_ONLY_FILE), 100);
  }

  @Test
  public void positionedReadNoCache() throws Exception {
    createUfsInStream(ReadType.NO_CACHE);
    mUfsInputStream.readFully(0, new byte[FILE_LEN]);
    URIStatus statusUfsOnlyFile = mFileSystem.getStatus(new AlluxioURI(UFS_ONLY_FILE));
    Assert.assertEquals(0, statusUfsOnlyFile.getInAlluxioPercentage());
  }
}
