alluxio-master-2.x/tests/src/test/java/alluxio/client/cli/fs/command/CopyFromLocalCommandIntegrationTest.java
/*
 * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
 * (the "License"). You may not use this work except in compliance with the License, which is
 * available at www.apache.org/licenses/LICENSE-2.0
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied, as more fully set forth in the License.
 *
 * See the NOTICE file distributed with this work for information regarding copyright ownership.
 */

package alluxio.client.cli.fs.command;

import static org.hamcrest.CoreMatchers.containsString;

import alluxio.AlluxioURI;
import alluxio.ConfigurationRule;
import alluxio.SystemPropertyRule;
import alluxio.client.WriteType;
import alluxio.client.cli.fs.AbstractFileSystemShellTest;
import alluxio.client.cli.fs.FileSystemShellUtilsTest;
import alluxio.client.file.FileInStream;
import alluxio.client.file.URIStatus;
import alluxio.conf.Configuration;
import alluxio.conf.PropertyKey;
import alluxio.exception.AlluxioException;
import alluxio.exception.ExceptionMessage;
import alluxio.grpc.DeletePOptions;
import alluxio.grpc.OpenFilePOptions;
import alluxio.grpc.ReadPType;
import alluxio.testutils.LocalAlluxioClusterResource;
import alluxio.util.io.BufferUtils;

import org.apache.commons.io.FileUtils;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

import java.io.Closeable;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

/**
 * Tests for copyFromLocal command.
 */
public final class CopyFromLocalCommandIntegrationTest extends AbstractFileSystemShellTest {
  /** Rule to create a new temporary folder during each test. */
  @Rule
  public TemporaryFolder mTestFolder = new TemporaryFolder();

  @Test
  public void copyFromLocalFileToDstPath() throws IOException, AlluxioException {
    String dataString = "copyFromLocalFileToDstPathTest";
    byte[] data = dataString.getBytes();
    File localDir = new File(sLocalAlluxioCluster.getAlluxioHome() + "/localDir");
    localDir.mkdir();
    File localFile = generateFileContent("/localDir/testFile", data);
    sFsShell.run("mkdir", "/dstDir");
    sFsShell.run("copyFromLocal", localFile.getPath(), "/dstDir");

    AlluxioURI uri = new AlluxioURI("/dstDir/testFile");
    URIStatus status = sFileSystem.getStatus(uri);
    Assert.assertNotNull(status);
    byte[] read = readContent(uri, data.length);
    Assert.assertEquals(new String(read), dataString);
  }

  @Test
  public void copyFromLocalDir() throws IOException, AlluxioException {
    // Copy a directory from local to Alluxio filesystem, which the destination uri was not created
    // before.
    File srcOuterDir = new File(sLocalAlluxioCluster.getAlluxioHome() + "/outerDir");
    File srcInnerDir = new File(sLocalAlluxioCluster.getAlluxioHome() + "/outerDir/innerDir");
    File emptyDir = new File(sLocalAlluxioCluster.getAlluxioHome() + "/outerDir/emptyDir");
    srcOuterDir.mkdir();
    srcInnerDir.mkdir();
    emptyDir.mkdir();
    generateFileContent("/outerDir/srcFile1", BufferUtils.getIncreasingByteArray(10));
    generateFileContent("/outerDir/innerDir/srcFile2", BufferUtils.getIncreasingByteArray(10));
    int ret = sFsShell.run("copyFromLocal", srcOuterDir.getPath() + "/", "/dstDir");
    Assert.assertEquals(0, ret);
    AlluxioURI dstURI1 = new AlluxioURI("/dstDir/srcFile1");
    AlluxioURI dstURI2 = new AlluxioURI("/dstDir/innerDir/srcFile2");
    AlluxioURI dstURI3 = new AlluxioURI("/dstDir/emptyDir");
    Assert.assertNotNull(sFileSystem.getStatus(dstURI1));
    Assert.assertNotNull(sFileSystem.getStatus(dstURI2));
    Assert.assertNotNull(sFileSystem.getStatus(dstURI3));
  }

  @Test
  public void copyFromLocalDirToExistingFile() throws IOException, AlluxioException {
    // Copy a directory from local to a file which exists in Alluxio filesystem. This case should
    // fail.
    File localDir = new File(sLocalAlluxioCluster.getAlluxioHome() + "/localDir");
    File innerDir = new File(sLocalAlluxioCluster.getAlluxioHome() + "/localDir/innerDir");
    localDir.mkdir();
    innerDir.mkdir();
    generateFileContent("/localDir/srcFile", BufferUtils.getIncreasingByteArray(10));
    sFileSystem.createFile(new AlluxioURI("/dstFile")).close();
    int ret = sFsShell.run("copyFromLocal", localDir.getPath(), "/dstFile");
    Assert.assertEquals(-1, ret);
    Assert.assertFalse(sFileSystem.getStatus(new AlluxioURI("/dstFile")).isFolder());
    Assert.assertFalse(sFileSystem.exists(new AlluxioURI("/dstFile/innerDir")));
  }

  @Test
  public void copyFromLocalDirToExistingDir() throws IOException, AlluxioException {
    // Copy a directory from local to Alluxio filesystem, which the destination uri has been
    // created before.
    File srcOuterDir = new File(sLocalAlluxioCluster.getAlluxioHome() + "/outerDir");
    File srcInnerDir = new File(sLocalAlluxioCluster.getAlluxioHome() + "/outerDir/innerDir");
    File emptyDir = new File(sLocalAlluxioCluster.getAlluxioHome() + "/outerDir/emptyDir");
    srcOuterDir.mkdir();
    srcInnerDir.mkdir();
    emptyDir.mkdir();
    generateFileContent("/outerDir/srcFile1", BufferUtils.getIncreasingByteArray(10));
    generateFileContent("/outerDir/innerDir/srcFile2", BufferUtils.getIncreasingByteArray(10));
    // Copying a directory to a destination directory which exists and doesn't contain the copied
    // directory.
    sFileSystem.createDirectory(new AlluxioURI("/dstDir"));
    int ret = sFsShell.run("copyFromLocal", srcOuterDir.getPath(), "/dstDir");
    Assert.assertEquals(0, ret);
    AlluxioURI dstURI1 = new AlluxioURI("/dstDir/srcFile1");
    AlluxioURI dstURI2 = new AlluxioURI("/dstDir/innerDir/srcFile2");
    AlluxioURI dstURI3 = new AlluxioURI("/dstDir/emptyDir");
    Assert.assertNotNull(sFileSystem.getStatus(dstURI1));
    Assert.assertNotNull(sFileSystem.getStatus(dstURI2));
    Assert.assertNotNull(sFileSystem.getStatus(dstURI3));

    // Copying a directory to a destination directory which exists and does contain the copied
    // directory.
    sFileSystem.createDirectory(new AlluxioURI("/dstDir1"));
    sFileSystem.createDirectory(new AlluxioURI("/dstDir1/innerDir"));
    int ret1 = sFsShell.run("copyFromLocal", srcOuterDir.getPath(), "/dstDir1");
    Assert.assertEquals(-1, ret1);
    dstURI1 = new AlluxioURI("/dstDir1/srcFile1");
    dstURI2 = new AlluxioURI("/dstDir1/innerDir/srcFile2");
    dstURI3 = new AlluxioURI("/dstDir1/emptyDir");
    Assert.assertNotNull(sFileSystem.getStatus(dstURI1));
    // The directory already exists. But the sub directory shouldn't be copied.
    Assert.assertFalse(sFileSystem.exists(dstURI2));
    Assert.assertNotNull(sFileSystem.getStatus(dstURI3));
  }

  @Test
  @LocalAlluxioClusterResource.Config(
      confParams = {PropertyKey.Name.WORKER_NETWORK_NETTY_WATERMARK_HIGH, "1.0"})
  public void copyFromLocalLarge() throws IOException, AlluxioException {
    File testFile = new File(sLocalAlluxioCluster.getAlluxioHome() + "/testFile");
    testFile.createNewFile();
    FileOutputStream fos = new FileOutputStream(testFile);
    byte[] toWrite = BufferUtils.getIncreasingByteArray(SIZE_BYTES);
    fos.write(toWrite);
    fos.close();
    sFsShell.run("copyFromLocal", testFile.getAbsolutePath(), "/testFile");
    Assert.assertEquals(
        getCommandOutput(new String[] {"copyFromLocal", testFile.getAbsolutePath(),
            "/testFile"}),
        mOutput.toString());
    AlluxioURI uri = new AlluxioURI("/testFile");
    URIStatus status = sFileSystem.getStatus(uri);
    Assert.assertNotNull(status);
    Assert.assertEquals(SIZE_BYTES, status.getLength());

    try (FileInStream tfis = sFileSystem.openFile(uri,
        OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build())) {
      byte[] read = new byte[SIZE_BYTES];
      tfis.read(read);
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(SIZE_BYTES, read));
    }
  }

  @Test
  public void copyFromLocalOverwrite() throws Exception {
    // This tests makes sure copyFromLocal will not overwrite an existing Alluxio file
    final int LEN1 = 10;
    final int LEN2 = 20;
    File testFile1 = generateFileContent("/testFile1", BufferUtils.getIncreasingByteArray(LEN1));
    File testFile2 = generateFileContent("/testFile2", BufferUtils.getIncreasingByteArray(LEN2));
    AlluxioURI alluxioFilePath = new AlluxioURI("/testFile");

    // Write the first file
    String[] cmd1 = {"copyFromLocal", testFile1.getPath(), alluxioFilePath.getPath()};
    sFsShell.run(cmd1);
    Assert.assertEquals(getCommandOutput(cmd1), mOutput.toString());
    mOutput.reset();
    Assert.assertTrue(BufferUtils
        .equalIncreasingByteArray(LEN1, readContent(alluxioFilePath, LEN1)));

    // Write the second file to the same location, which should cause an exception
    String[] cmd2 = {"copyFromLocal", testFile2.getPath(), alluxioFilePath.getPath()};
    Assert.assertEquals(-1, sFsShell.run(cmd2));
    Assert.assertThat(mOutput.toString(), containsString(
        ExceptionMessage.CANNOT_OVERWRITE_FILE_WITHOUT_OVERWRITE.getMessage(
            alluxioFilePath.getPath())));
    // Make sure the original file is intact
    Assert.assertTrue(BufferUtils
        .equalIncreasingByteArray(LEN1, readContent(alluxioFilePath, LEN1)));
  }

  @Test
  public void copyFromLocal() throws IOException, AlluxioException {
    File testDir = new File(sLocalAlluxioCluster.getAlluxioHome() + "/testDir");
    testDir.mkdir();
    File testDirInner = new File(sLocalAlluxioCluster.getAlluxioHome() + "/testDir/testDirInner");
    testDirInner.mkdir();
    File testFile =
        generateFileContent("/testDir/testFile", BufferUtils.getIncreasingByteArray(10));
    generateFileContent("/testDir/testDirInner/testFile2",
        BufferUtils.getIncreasingByteArray(10, 20));

    sFsShell.run("copyFromLocal", testFile.getParent(), "/testDir");
    Assert.assertThat(mOutput.toString(), containsString(
        getCommandOutput(new String[]{"copyFromLocal", testFile.getParent(), "/testDir"})));
    AlluxioURI uri1 = new AlluxioURI("/testDir/testFile");
    AlluxioURI uri2 = new AlluxioURI("/testDir/testDirInner/testFile2");
    URIStatus status1 = sFileSystem.getStatus(uri1);
    URIStatus status2 = sFileSystem.getStatus(uri2);
    Assert.assertNotNull(status1);
    Assert.assertNotNull(status2);
    Assert.assertEquals(10, status1.getLength());
    Assert.assertEquals(20, status2.getLength());
    byte[] read = readContent(uri1, 10);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, read));
    read = readContent(uri2, 20);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, 20, read));
  }

  @Test
  public void copyFromLocalMustCacheThenCacheThrough() throws Exception {
    File file = mTestFolder.newFile();
    try (Closeable c = new ConfigurationRule(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT,
        WriteType.MUST_CACHE.toString(), Configuration.modifiableGlobal()).toResource()) {
      Assert.assertEquals(0, sFsShell.run("copyFromLocal", file.getAbsolutePath(), "/test"));
    }
    try (Closeable c = new ConfigurationRule(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT,
        WriteType.CACHE_THROUGH.toString(), Configuration.modifiableGlobal()).toResource()) {
      mOutput.reset();
      sFsShell.run("copyFromLocal", file.getAbsolutePath(), "/test");
    }
    Assert.assertThat(mOutput.toString(),
        containsString(
            ExceptionMessage.CANNOT_OVERWRITE_FILE_WITHOUT_OVERWRITE.getMessage("/test")));
  }

  @Test
  public void copyFromLocalTestWithFullURI() throws IOException, AlluxioException {
    File testFile = generateFileContent("/srcFileURI", BufferUtils.getIncreasingByteArray(10));
    String alluxioURI = "alluxio://" + sLocalAlluxioCluster.getHostname() + ":"
        + sLocalAlluxioCluster.getMasterRpcPort() + "/destFileURI";
    // when
    sFsShell.run("copyFromLocal", testFile.getPath(), alluxioURI);
    String cmdOut =
        getCommandOutput(new String[]{"copyFromLocal", testFile.getPath(), alluxioURI});
    // then
    Assert.assertEquals(cmdOut, mOutput.toString());
    AlluxioURI uri = new AlluxioURI("/destFileURI");
    URIStatus status = sFileSystem.getStatus(uri);
    Assert.assertEquals(10L, status.getLength());
    byte[] read = readContent(uri, 10);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, read));
  }

  @Test
  public void copyFromLocalWildcardExistingDir() throws IOException, AlluxioException {
    String testDir = FileSystemShellUtilsTest.resetLocalFileHierarchy(sLocalAlluxioCluster);
    sFileSystem.createDirectory(new AlluxioURI("/testDir"));
    int ret = sFsShell.run("copyFromLocal", testDir + "/*/foo*", "/testDir");
    Assert.assertEquals(0, ret);
    Assert.assertTrue(fileExists(new AlluxioURI("/testDir/foobar1")));
    Assert.assertTrue(fileExists(new AlluxioURI("/testDir/foobar2")));
    Assert.assertTrue(fileExists(new AlluxioURI("/testDir/foobar3")));
  }

  @Test
  public void copyFromLocalWildcardHier() throws IOException {
    String testDir = FileSystemShellUtilsTest.resetLocalFileHierarchy(sLocalAlluxioCluster);
    int ret = sFsShell.run("copyFromLocal", testDir + "/*", "/testDir");
    Assert.assertEquals(0, ret);
    Assert.assertTrue(fileExists(new AlluxioURI("/testDir/foo/foobar1")));
    Assert.assertTrue(fileExists(new AlluxioURI("/testDir/foo/foobar2")));
    Assert.assertTrue(fileExists(new AlluxioURI("/testDir/bar/foobar3")));
    Assert.assertTrue(fileExists(new AlluxioURI("/testDir/foobar4")));
  }

  @Test
  public void copyFromLocalWildcardNotDir() throws IOException, AlluxioException {
    String localTestDir = FileSystemShellUtilsTest.resetFileHierarchy(sFileSystem);
    String alluxioTestDir = FileSystemShellUtilsTest.resetFileHierarchy(sFileSystem);
    int ret = sFsShell.run("copyFromLocal", localTestDir + "/*/foo*", alluxioTestDir + "/foobar4");
    Assert.assertEquals(-1, ret);
  }

  @Test
  public void copyFromLocalWildcard() throws IOException {
    String testDir = FileSystemShellUtilsTest.resetLocalFileHierarchy(sLocalAlluxioCluster);
    int ret = sFsShell.run("copyFromLocal", testDir + "/*/foo*", "/testDir");
    Assert.assertEquals(0, ret);
    Assert.assertTrue(fileExists(new AlluxioURI("/testDir/foobar1")));
    Assert.assertTrue(fileExists(new AlluxioURI("/testDir/foobar2")));
    Assert.assertTrue(fileExists(new AlluxioURI("/testDir/foobar3")));
    Assert.assertFalse(fileExists(new AlluxioURI("/testDir/foobar4")));
  }

  @Test
  public void copyFromLocalRelativePath() throws Exception {
    String version = System.getProperty("java.version");
    if (version.startsWith("11")) {
      // Breaks on Java 11+ because of https://bugs.openjdk.java.net/browse/JDK-8202127
      return;
    }

    HashMap<String, String> sysProps = new HashMap<>();
    sysProps.put("user.dir", mTestFolder.getRoot().getAbsolutePath());
    try (Closeable p = new SystemPropertyRule(sysProps).toResource()) {
      File localDir = mTestFolder.newFolder("testDir");
      generateRelativeFileContent(localDir.getPath() + "/testFile",
              BufferUtils.getIncreasingByteArray(10));
      int ret = sFsShell.run("copyFromLocal", "testDir/testFile", "/testFile");
      Assert.assertEquals(0, ret);
      Assert.assertTrue(fileExists(new AlluxioURI(("/testFile"))));
    }
  }

  @Test(timeout = 30000)
  public void copyFromLargeLocalDirectory() throws Exception {
    final String testDir = FileSystemShellUtilsTest.TEST_DIR;
    final String filePathFormat = "file_%d";
    final File src = new File(sLocalAlluxioCluster.getAlluxioHome() + testDir);
    FileUtils.deleteDirectory(src);
    src.mkdir();
    ExecutorService executor = Executors.newFixedThreadPool(
        Runtime.getRuntime().availableProcessors() * 2);
    final int numFiles = 2_000;
    for (int i = 0; i < numFiles; i++) {
      final int fileId = i;
      executor.submit(() -> {
        try {
          new File(src, String.format(filePathFormat, fileId)).createNewFile();
        } catch (IOException e) {
          // Ignore errors here, if the file cannot be created, asserts below will fail.
        }
      });
    }
    executor.shutdown();
    executor.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS);
    File[] files = src.listFiles();
    Assert.assertNotNull(files);
    Assert.assertEquals(numFiles, files.length);

    AlluxioURI dst = new AlluxioURI(testDir);
    if (sFileSystem.exists(dst)) {
      sFileSystem.delete(dst, DeletePOptions.newBuilder().setRecursive(true).build());
    }
    sFileSystem.createDirectory(dst);

    int ret = sFsShell.run("copyFromLocal", src.getAbsolutePath(), testDir);
    Assert.assertEquals(0, ret);
    for (int i = 0; i < numFiles; i++) {
      Assert.assertTrue(fileExists(dst.join(String.format(filePathFormat, i))));
    }
  }

  @Test
  public void parseOption() throws Exception {
    String testDir = FileSystemShellUtilsTest.resetLocalFileHierarchy(sLocalAlluxioCluster);
    int ret = sFsShell.run("copyFromLocal", "--thread", "1", "--buffersize", "1024",
        testDir + "/*/foo*", "/testDir");
    Assert.assertEquals(0, ret);
  }
}
