alluxio-master-2.x/tests/src/test/java/alluxio/client/cli/fs/command/LsCommandIntegrationTest.java
/*
 * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
 * (the "License"). You may not use this work except in compliance with the License, which is
 * available at www.apache.org/licenses/LICENSE-2.0
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied, as more fully set forth in the License.
 *
 * See the NOTICE file distributed with this work for information regarding copyright ownership.
 */

package alluxio.client.cli.fs.command;

import static org.hamcrest.Matchers.matchesPattern;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;

import alluxio.AlluxioURI;
import alluxio.client.cli.fs.AbstractFileSystemShellTest;
import alluxio.client.cli.fs.FileSystemShellUtilsTest;
import alluxio.client.file.FileSystem;
import alluxio.client.file.FileSystemContext;
import alluxio.client.file.FileSystemTestUtils;
import alluxio.client.file.URIStatus;
import alluxio.conf.Configuration;
import alluxio.conf.PropertyKey;
import alluxio.exception.AlluxioException;
import alluxio.grpc.SetAttributePOptions;
import alluxio.grpc.WritePType;
import alluxio.security.user.TestUserState;
import alluxio.testutils.LocalAlluxioClusterResource;

import org.junit.Test;

import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;

/**
 * Tests for ls command.
 */
@LocalAlluxioClusterResource.ServerConfig(
    confParams = {PropertyKey.Name.SECURITY_AUTHORIZATION_PERMISSION_ENABLED, "false",
        PropertyKey.Name.SECURITY_AUTHENTICATION_TYPE, "NOSASL",
        PropertyKey.Name.MASTER_FILE_ACCESS_TIME_UPDATE_PRECISION, "0"})
public final class LsCommandIntegrationTest extends AbstractFileSystemShellTest {
  private static final String FILE_NEW_CONTENT = "testfoobarlongerthantheoriginalfile";

  // Helper function to create a set of files in the file system
  private void createFiles(String user) throws Exception {
    FileSystem fs = sFileSystem;
    if (user != null) {
      fs = sLocalAlluxioCluster.getClient(FileSystemContext
          .create(new TestUserState(user, Configuration.global()).getSubject(),
              Configuration.global()));
    }
    FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileA", WritePType.MUST_CACHE, 10);
    FileSystemTestUtils
        .createByteFile(fs, "/testRoot/testDir/testFileB", WritePType.MUST_CACHE, 20);
    FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileC", WritePType.THROUGH, 30);
  }

  /**
   * Tests ls command when security is not enabled.
   */
  @Test
  public void lsNoAcl() throws Exception {
    createFiles(null);
    sFsShell.run("ls", "--sort", "path", "/testRoot");
    checkOutput(
        "              1   NOT_PERSISTED .+ .+  DIR /testRoot/testDir",
        "             10   NOT_PERSISTED .+ .+ 100% /testRoot/testFileA",
        "             30       PERSISTED .+ .+   0% /testRoot/testFileC");
  }

  /**
   * Tests ls command when arguments are multiple directories and security is not enabled.
   */
  @Test
  public void lsMultipleDirs() throws Exception {
    createFiles(null);
    sFsShell.run("ls", "--sort", "path", "/testRoot", "/testRoot/testDir/");
    checkOutput(
        "              1   NOT_PERSISTED .+ .+  DIR /testRoot/testDir",
        "             10   NOT_PERSISTED .+ .+ 100% /testRoot/testFileA",
        "             30       PERSISTED .+ .+   0% /testRoot/testFileC",
        "             20   NOT_PERSISTED .+ .+ 100% /testRoot/testDir/testFileB");
  }

  /**
   * Tests ls -h command when security is not enabled.
   */
  @Test
  public void lsHumanReadable() throws Exception {
    createFiles(null);
    sFsShell.run("ls", "-h", "--sort", "path", "/testRoot");
    checkOutput(
        "              1   NOT_PERSISTED .+ .+  DIR /testRoot/testDir",
        "            10B   NOT_PERSISTED .+ .+ 100% /testRoot/testFileA",
        "            30B       PERSISTED .+ .+   0% /testRoot/testFileC");
  }

  /**
   * Tests ls -p command when security is not enabled.
   */
  @Test
  public void lsPinned() throws Exception {
    createFiles(null);
    AlluxioURI fileURI1 = new AlluxioURI("/testRoot/testDir/testFileB");
    AlluxioURI fileURI2 = new AlluxioURI("/testRoot/testFileA");
    sFileSystem.setAttribute(fileURI1,
        SetAttributePOptions.newBuilder().setPinned(true).build());
    sFileSystem.setAttribute(fileURI2,
        SetAttributePOptions.newBuilder().setPinned(true).build());
    sFsShell.run("ls", "-pR", "--sort", "path", "/testRoot");
    checkOutput(
        "             20   NOT_PERSISTED .+ .+ 100% /testRoot/testDir/testFileB",
        "             10   NOT_PERSISTED .+ .+ 100% /testRoot/testFileA");
  }

  /**
   * Tests ls -d command when security is not enabled.
   */
  @Test
  public void lsDirectoryAsPlainFileNoAcl() throws Exception {
    createFiles(null);
    sFsShell.run("ls", "-d", "/testRoot");
    checkOutput("              3       PERSISTED .+ .+  DIR /testRoot");
  }

  /**
   * Tests ls -d command on root directory when security is not enabled.
   */
  @Test
  public void lsRootNoAcl() throws Exception {
    sFsShell.run("ls", "-d", "/");
    checkOutput("              0       PERSISTED .+ .+  DIR /    ");
  }

  /**
   * Tests ls command with wildcard when security is not enabled.
   */
  @Test
  public void lsWildcardNoAcl() throws Exception {
    String testDir = FileSystemShellUtilsTest.resetFileHierarchy(sFileSystem);

    sFsShell.run("ls", "--sort", "path", testDir + "/*/foo*");
    checkOutput(
        "             30   NOT_PERSISTED .+ .+ 100% /testDir/bar/foobar3",
        "             10   NOT_PERSISTED .+ .+ 100% /testDir/foo/foobar1",
        "             20   NOT_PERSISTED .+ .+ 100% /testDir/foo/foobar2");

    mOutput.reset();

    sFsShell.run("ls", "--sort", "path", testDir + "/*");
    checkOutput(
        "             30   NOT_PERSISTED .+ .+ 100% /testDir/bar/foobar3",
        "             10   NOT_PERSISTED .+ .+ 100% /testDir/foo/foobar1",
        "             20   NOT_PERSISTED .+ .+ 100% /testDir/foo/foobar2",
        "             40   NOT_PERSISTED .+ .+ 100% /testDir/foobar4");
  }

  /**
   * Tests ls -R command with wildcard when security is not enabled.
   */
  @Test
  public void lsrNoAcl() throws Exception {
    createFiles(null);
    sFsShell.run("ls", "-R", "--sort", "path", "/testRoot");
    checkOutput(
        "              1   NOT_PERSISTED .+ .+  DIR /testRoot/testDir",
        "             20   NOT_PERSISTED .+ .+ 100% /testRoot/testDir/testFileB",
        "             10   NOT_PERSISTED .+ .+ 100% /testRoot/testFileA",
        "             30       PERSISTED .+ .+   0% /testRoot/testFileC");
  }

  /**
   * Tests ls command with a file where the file name includes a specifier character.
   */
  @Test
  public void lsWithFormatSpecifierCharacter() throws IOException, AlluxioException {
    String fileName = "/localhost%2C61764%2C1476207067267..meta.1476207073442.meta";
    FileSystemTestUtils.createByteFile(sFileSystem, fileName, WritePType.MUST_CACHE, 10);
    sFsShell.run("ls", "/");
    // CHECKSTYLE.OFF: LineLengthExceed - Improve readability
    checkOutput("             10   NOT_PERSISTED .+ .+ 100% /localhost%2C61764%2C1476207067267..meta.1476207073442.meta");
    // CHECKSTYLE.ON: LineLengthExceed
  }

  /**
   * Tests ls command with sort by path option.
   */
  @Test
  public void lsWithSortByPath() throws IOException, AlluxioException {
    FileSystemTestUtils
            .createByteFile(sFileSystem, "/testRoot/testLongFile", WritePType.MUST_CACHE, 100);
    FileSystemTestUtils
            .createByteFile(sFileSystem, "/testRoot/testFileZ", WritePType.MUST_CACHE, 10);
    FileSystemTestUtils
            .createByteFile(sFileSystem, "/testRoot/testFileA", WritePType.MUST_CACHE, 50);
    sFsShell.run("ls", "--sort", "path", "/testRoot");
    checkOutput(
        "             50   NOT_PERSISTED .+ .+ 100% /testRoot/testFileA",
        "             10   NOT_PERSISTED .+ .+ 100% /testRoot/testFileZ",
        "            100   NOT_PERSISTED .+ .+ 100% /testRoot/testLongFile");
  }

  /**
   * Tests ls command with sort by access time option.
   */
  @Test
  public void lsWithSortByAccessTime() throws IOException, AlluxioException {
    String oldFileRecentlyAccessed = "/testRoot/testFileRecent";
    String oldFileName = "/testRoot/testFile";
    FileSystemTestUtils
        .createByteFile(sFileSystem, oldFileRecentlyAccessed, WritePType.MUST_CACHE, 10);
    FileSystemTestUtils
        .createByteFile(sFileSystem, oldFileName, WritePType.MUST_CACHE, 10);

    FileSystemTestUtils.loadFile(sFileSystem, oldFileRecentlyAccessed);
    sFsShell.run("ls", "--sort", "lastAccessTime", "/testRoot");
    checkOutput(
        "             10   NOT_PERSISTED .+ .+ 100% " + oldFileName,
        "             10   NOT_PERSISTED .+ .+ 100% " + oldFileRecentlyAccessed);
  }

  /**
   * Tests ls command with sort by creation time option.
   */
  @Test
  public void lsWithSortByCreationTime() throws IOException, AlluxioException {
    String oldFileRecentlyAccessed = "/testRoot/testFileRecent";
    String oldFileName = "/testRoot/testFile";
    FileSystemTestUtils
        .createByteFile(sFileSystem, oldFileRecentlyAccessed, WritePType.MUST_CACHE, 10);
    FileSystemTestUtils
        .createByteFile(sFileSystem, oldFileName, WritePType.MUST_CACHE, 10);

    FileSystemTestUtils.loadFile(sFileSystem, oldFileRecentlyAccessed);
    sFsShell.run("ls", "--sort", "creationTime", "/testRoot");
    checkOutput(
        "             10   NOT_PERSISTED .+ .+ 100% " + oldFileRecentlyAccessed,
        "             10   NOT_PERSISTED .+ .+ 100% " + oldFileName);
  }

  /**
   * Tests ls command with sort by size option.
   */
  @Test
  public void lsWithSortBySize() throws IOException, AlluxioException {
    FileSystemTestUtils
            .createByteFile(sFileSystem, "/testRoot/testFileA", WritePType.MUST_CACHE, 50, 50);
    FileSystemTestUtils
            .createByteFile(sFileSystem, "/testRoot/testFileZ", WritePType.MUST_CACHE, 10, 10);
    FileSystemTestUtils
            .createByteFile(sFileSystem, "/testRoot/testLongFile", WritePType.MUST_CACHE, 100, 100);
    sFsShell.run("ls", "--sort", "size", "/testRoot");
    checkOutput(
        "             10   NOT_PERSISTED .+ .+ 100% /testRoot/testFileZ",
        "             50   NOT_PERSISTED .+ .+ 100% /testRoot/testFileA",
        "            100   NOT_PERSISTED .+ .+ /testRoot/testLongFile");
  }

  /**
   * Tests ls command with sort by size and reverse order option.
   */
  @Test
  public void lsWithSortBySizeAndReverse() throws IOException, AlluxioException {
    FileSystemTestUtils
            .createByteFile(sFileSystem, "/testRoot/testFileA", WritePType.MUST_CACHE, 50, 50);
    FileSystemTestUtils
            .createByteFile(sFileSystem, "/testRoot/testFileZ", WritePType.MUST_CACHE, 10, 10);
    FileSystemTestUtils
            .createByteFile(sFileSystem, "/testRoot/testLongFile", WritePType.MUST_CACHE, 100, 100);
    sFsShell.run("ls", "--sort", "size", "-r", "/testRoot");
    checkOutput(
        "            100   NOT_PERSISTED .+ .+ 100% /testRoot/testLongFile",
        "             50   NOT_PERSISTED .+ .+ 100% /testRoot/testFileA",
        "             10   NOT_PERSISTED .+ .+ 100% /testRoot/testFileZ");
  }

  /**
   * Tests ls command with an invalid sort option.
   */
  @Test
  public void lsWithInvalidSortOption() throws IOException, AlluxioException {
    FileSystemTestUtils.createByteFile(sFileSystem, "/testRoot/testFileA",
        WritePType.MUST_CACHE, 50, 50);
    sFsShell.run("ls", "--sort", "unknownfield", "/testRoot");
    String expected = "Invalid sort option `unknownfield` for --sort\n";
    assertEquals(expected, mOutput.toString());
  }

  /**
   * Tests ls command with reverse sort order option.
   */
  @Test
  public void lsReverseWithoutSort() throws IOException, AlluxioException {
    FileSystemTestUtils
            .createByteFile(sFileSystem, "/testRoot/testFileA", WritePType.MUST_CACHE, 50, 50);
    FileSystemTestUtils
            .createByteFile(sFileSystem, "/testRoot/testFileZ", WritePType.MUST_CACHE, 10, 10);
    FileSystemTestUtils
            .createByteFile(sFileSystem, "/testRoot/testLongFile", WritePType.MUST_CACHE, 100, 100);
    sFsShell.run("ls", "-r", "--sort", "path", "/testRoot");
    checkOutput(
        "            100   NOT_PERSISTED .+ .+ 100% /testRoot/testLongFile",
        "             10   NOT_PERSISTED .+ .+ 100% /testRoot/testFileZ",
        "             50   NOT_PERSISTED .+ .+ 100% /testRoot/testFileA");
  }

  @Test
  public void lsAfterForceMasterSync() throws Exception {
    // Create a file in the UFS and write some bytes into it
    String testFilePath = "/testPersist/testFile";
    int size = 100;
    FileSystemTestUtils.createByteFile(sFileSystem, testFilePath, WritePType.THROUGH, size);
    assertTrue(sFileSystem.getStatus(new AlluxioURI(testFilePath)).isPersisted());

    // Update the file with a smaller size in the UFS directly
    String rootUfsDir = sLocalAlluxioCluster.getClient().getConf()
        .get(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS).toString();
    File ufsFile = new File(rootUfsDir + testFilePath);
    ufsFile.delete();
    PrintWriter out = new PrintWriter(ufsFile);
    String newContent = FILE_NEW_CONTENT;
    out.print(newContent);
    out.close();

    // Alluxio has not realized the out-of-band update so the file metadata stays unchanged
    AlluxioURI uri = new AlluxioURI(testFilePath);
    URIStatus uriStatus = sFileSystem.getStatus(uri);
    long length = uriStatus.getLength();
    assertEquals(size, length);

    // Read the file by executing "alluxio fs cat <FILE_PATH>".
    // Exception should be thrown when the worker try reading bytes and returned value is -1 here
    // After that the master is forced to sync
    int ret = sFsShell.run("cat", testFilePath);
    assertEquals(-1, ret);
    String errMsg = String.format("expected to be %s bytes, but only %s bytes are available",
        size, newContent.getBytes().length);
    assertTrue(mOutput.toString().contains(errMsg));
    mOutput.reset();

    // A failed read will force refresh metadata so now the updated value is seen
    sFsShell.run("ls", "/testPersist/");
    checkOutput("\\s+" + newContent.getBytes().length + "\\s+PERSISTED (.*) 0% " + testFilePath);
  }

  @Test
  public void recursiveLsAfterForceMasterSync() throws Exception {
    // Create a file in the UFS and write some bytes into it
    String testFilePath = "/testPersist/testFile";
    int size = 100;
    FileSystemTestUtils.createByteFile(sFileSystem, testFilePath, WritePType.THROUGH, size);
    assertTrue(sFileSystem.getStatus(new AlluxioURI(testFilePath)).isPersisted());

    // Update the file with a smaller size in the UFS directly
    String rootUfsDir = sLocalAlluxioCluster.getClient().getConf()
        .get(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS).toString();
    File ufsFile = new File(rootUfsDir + testFilePath);
    ufsFile.delete();
    PrintWriter out = new PrintWriter(ufsFile);
    String newContent = FILE_NEW_CONTENT;
    out.print(newContent);
    out.close();

    // Alluxio has not realized the out-of-band update so the file metadata stays unchanged
    AlluxioURI uri = new AlluxioURI(testFilePath);
    URIStatus uriStatus = sFileSystem.getStatus(uri);
    long length = uriStatus.getLength();
    assertEquals(size, length);

    // Read the file by executing "alluxio fs cat <FILE_PATH>".
    // Exception should be thrown when the worker try reading bytes and returned value is -1 here
    // After that the master is forced to sync
    int ret = sFsShell.run("cat", testFilePath);
    assertEquals(-1, ret);
    String errMsg = String.format("expected to be %s bytes, but only %s bytes are available",
        size, newContent.getBytes().length);
    assertTrue(mOutput.toString().contains(errMsg));
    mOutput.reset();

    // A failed read will force refresh metadata so now the updated value is seen
    sFsShell.run("ls", "-R", "/");
    checkOutput("\\s+1\\s+ PERSISTED (.*) DIR /testPersist",
        "\\s+" + newContent.getBytes().length + "\\s+PERSISTED (.*) 0% " + testFilePath);
  }

  private void checkOutput(String... linePatterns) {
    String[] actualLines = mOutput.toString().split("\n");
    assertEquals("Output: " + mOutput.toString(), linePatterns.length, actualLines.length);
    for (int i = 0; i < linePatterns.length; i++) {
      assertThat("mOutput: " + mOutput.toString(), actualLines[i], matchesPattern(linePatterns[i]));
    }
  }
}
