alluxio-master-2.x/tests/src/test/java/alluxio/client/fs/CheckConsistencyIntegrationTest.java
/*
 * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
 * (the "License"). You may not use this work except in compliance with the License, which is
 * available at www.apache.org/licenses/LICENSE-2.0
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied, as more fully set forth in the License.
 *
 * See the NOTICE file distributed with this work for information regarding copyright ownership.
 */

package alluxio.client.fs;

import alluxio.AlluxioURI;
import alluxio.AuthenticatedUserRule;
import alluxio.client.file.FileSystem;
import alluxio.conf.Configuration;
import alluxio.conf.PropertyKey;
import alluxio.grpc.CreateDirectoryPOptions;
import alluxio.grpc.CreateFilePOptions;
import alluxio.grpc.WritePType;
import alluxio.master.file.FileSystemMaster;
import alluxio.master.file.contexts.CheckConsistencyContext;
import alluxio.testutils.BaseIntegrationTest;
import alluxio.testutils.LocalAlluxioClusterResource;
import alluxio.underfs.UnderFileSystem;
import alluxio.underfs.options.DeleteOptions;

import com.google.common.collect.Lists;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;

import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

/**
 * Integration test for
 * {@link FileSystemMaster#checkConsistency(AlluxioURI, CheckConsistencyContext)}.
 */
public class CheckConsistencyIntegrationTest extends BaseIntegrationTest {
  private static final AlluxioURI DIRECTORY = new AlluxioURI("/dir");
  private static final AlluxioURI FILE = new AlluxioURI("/dir/file");
  private static final String TEST_USER = "test";

  @Rule
  public LocalAlluxioClusterResource mLocalAlluxioClusterResource =
      new LocalAlluxioClusterResource.Builder().setProperty(PropertyKey.SECURITY_LOGIN_USERNAME,
          TEST_USER).build();

  @Rule
  public AuthenticatedUserRule mAuthenticatedUser = new AuthenticatedUserRule(TEST_USER,
      Configuration.global());

  private FileSystemMaster mFileSystemMaster;
  private FileSystem mFileSystem;

  @Before
  public final void before() throws Exception {
    mFileSystemMaster =
        mLocalAlluxioClusterResource.get().getLocalAlluxioMaster().getMasterProcess()
            .getMaster(FileSystemMaster.class);
    mFileSystem = FileSystem.Factory.create();
    CreateDirectoryPOptions dirOptions =
        CreateDirectoryPOptions.newBuilder().setWriteType(WritePType.CACHE_THROUGH).build();
    CreateFilePOptions fileOptions =
        CreateFilePOptions.newBuilder().setWriteType(WritePType.CACHE_THROUGH).build();
    mFileSystem.createDirectory(DIRECTORY, dirOptions);
    mFileSystem.createFile(FILE, fileOptions).close();
  }

  /**
   * Tests the {@link FileSystemMaster#checkConsistency(AlluxioURI, CheckConsistencyContext)} method
   * when all files are consistent.
   */
  @Test
  public void consistent() throws Exception {
    Assert.assertEquals(new ArrayList<AlluxioURI>(), mFileSystemMaster.checkConsistency(
        new AlluxioURI("/"), CheckConsistencyContext.defaults()));
  }

  /**
   * Tests the {@link FileSystemMaster#checkConsistency(AlluxioURI, CheckConsistencyContext)} method
   * when no files are consistent.
   */
  @Test
  public void inconsistent() throws Exception {
    String ufsDirectory = mFileSystem.getStatus(DIRECTORY).getUfsPath();
    UnderFileSystem ufs = UnderFileSystem.Factory.create(ufsDirectory,
        Configuration.global());
    ufs.deleteDirectory(ufsDirectory, DeleteOptions.defaults().setRecursive(true));

    List<AlluxioURI> expected = Lists.newArrayList(FILE, DIRECTORY);
    List<AlluxioURI> result = mFileSystemMaster.checkConsistency(new AlluxioURI("/"),
        CheckConsistencyContext.defaults());
    Collections.sort(expected);
    Collections.sort(result);
    Assert.assertEquals(expected, result);
  }

  /**
   * Tests the {@link FileSystemMaster#checkConsistency(AlluxioURI, CheckConsistencyContext)} method
   * when some files are consistent.
   */
  @Test
  public void partiallyInconsistent() throws Exception {
    String ufsFile = mFileSystem.getStatus(FILE).getUfsPath();
    UnderFileSystem ufs = UnderFileSystem.Factory.create(ufsFile, Configuration.global());
    ufs.deleteFile(ufsFile);
    List<AlluxioURI> expected = Lists.newArrayList(FILE);
    Assert.assertEquals(expected, mFileSystemMaster.checkConsistency(new AlluxioURI("/"),
        CheckConsistencyContext.defaults()));
  }

  /**
   * Tests the {@link FileSystemMaster#checkConsistency(AlluxioURI, CheckConsistencyContext)} method
   * when some files are consistent in a larger inode tree.
   */
  @Test
  public void largeTree() throws Exception {
    CreateDirectoryPOptions dirOptions =
        CreateDirectoryPOptions.newBuilder().setWriteType(WritePType.CACHE_THROUGH).build();
    CreateFilePOptions fileOptions =
        CreateFilePOptions.newBuilder().setWriteType(WritePType.CACHE_THROUGH).build();
    AlluxioURI nestedDir = DIRECTORY.join("/dir2");
    AlluxioURI topLevelFile = new AlluxioURI("/file");
    AlluxioURI thirdLevelFile = nestedDir.join("/file");
    mFileSystem.createDirectory(nestedDir, dirOptions);
    mFileSystem.createFile(topLevelFile, fileOptions).close();
    mFileSystem.createFile(thirdLevelFile, fileOptions).close();
    String ufsDirectory = mFileSystem.getStatus(nestedDir).getUfsPath();
    UnderFileSystem ufs = UnderFileSystem.Factory.create(ufsDirectory,
        Configuration.global());
    ufs.deleteDirectory(ufsDirectory, DeleteOptions.defaults().setRecursive(true));

    List<AlluxioURI> expected = Lists.newArrayList(nestedDir, thirdLevelFile);
    List<AlluxioURI> result = mFileSystemMaster.checkConsistency(new AlluxioURI("/"),
        CheckConsistencyContext.defaults());
    Collections.sort(expected);
    Collections.sort(result);
    Assert.assertEquals(expected, result);
  }

  /**
   * Tests the {@link FileSystemMaster#checkConsistency(AlluxioURI, CheckConsistencyContext)} method
   * when a file is not the correct size.
   */
  @Test
  public void incorrectFileSize() throws Exception {
    String ufsFile = mFileSystem.getStatus(FILE).getUfsPath();
    UnderFileSystem ufs = UnderFileSystem.Factory.create(ufsFile, Configuration.global());
    ufs.deleteFile(ufsFile);
    OutputStream out = ufs.create(ufsFile);
    out.write(1);
    out.close();
    List<AlluxioURI> expected = Lists.newArrayList(FILE);
    Assert.assertEquals(expected, mFileSystemMaster.checkConsistency(new AlluxioURI("/"),
        CheckConsistencyContext.defaults()));
  }

  /**
   * Tests the {@link FileSystemMaster#checkConsistency(AlluxioURI, CheckConsistencyContext)} method
   * when a directory does not exist as a directory in the under storage.
   */
  @Test
  public void notADirectory() throws Exception {
    String ufsDirectory = mFileSystem.getStatus(DIRECTORY).getUfsPath();
    UnderFileSystem ufs = UnderFileSystem.Factory.create(ufsDirectory,
        Configuration.global());
    ufs.deleteDirectory(ufsDirectory, DeleteOptions.defaults().setRecursive(true));
    ufs.create(ufsDirectory).close();
    List<AlluxioURI> expected = Lists.newArrayList(DIRECTORY, FILE);
    List<AlluxioURI> result = mFileSystemMaster.checkConsistency(new AlluxioURI("/"),
        CheckConsistencyContext.defaults());
    Collections.sort(expected);
    Collections.sort(result);
    Assert.assertEquals(expected, result);
  }

  @Test
  public void existsInUfsButNotAlluxio() throws Exception {
    String ufsDirectory = mFileSystem.getStatus(DIRECTORY).getUfsPath();
    UnderFileSystem ufs = UnderFileSystem.Factory.create(ufsDirectory,
        Configuration.global());
    String filename = "ufs_exists";
    String newPath = ufsDirectory + AlluxioURI.SEPARATOR + filename;
    ufs.mkdirs(newPath);
    List<AlluxioURI> expected = Lists.newArrayList(DIRECTORY.join(filename));
    Assert.assertEquals(expected, mFileSystemMaster.checkConsistency(DIRECTORY,
        CheckConsistencyContext.defaults()));
  }

  /**
   * Tests the {@link FileSystemMaster#checkConsistency(AlluxioURI, CheckConsistencyContext)} method
   * when a file does not exist as a file in the under storage.
   */
  @Test
  public void notAFile() throws Exception {
    String ufsFile = mFileSystem.getStatus(FILE).getUfsPath();
    UnderFileSystem ufs = UnderFileSystem.Factory.create(ufsFile, Configuration.global());
    ufs.deleteFile(ufsFile);
    ufs.mkdirs(ufsFile);
    List<AlluxioURI> expected = Lists.newArrayList(FILE);
    Assert.assertEquals(expected, mFileSystemMaster.checkConsistency(new AlluxioURI("/"),
        CheckConsistencyContext.defaults()));
  }

  /**
   * Tests the {@link FileSystemMaster#checkConsistency(AlluxioURI, CheckConsistencyContext)} method
   * when running on a file that is inconsistent.
   */
  @Test
  public void inconsistentFile() throws Exception {
    String ufsFile = mFileSystem.getStatus(FILE).getUfsPath();
    UnderFileSystem ufs = UnderFileSystem.Factory.create(ufsFile, Configuration.global());
    ufs.deleteFile(ufsFile);
    List<AlluxioURI> expected = Lists.newArrayList(FILE);
    Assert.assertEquals(expected, mFileSystemMaster.checkConsistency(FILE,
        CheckConsistencyContext.defaults()));
  }
}
