alluxio-master-2.x/tests/src/test/java/alluxio/server/ft/FileSystemMasterFaultToleranceIntegrationTest.java
/*
 * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
 * (the "License"). You may not use this work except in compliance with the License, which is
 * available at www.apache.org/licenses/LICENSE-2.0
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied, as more fully set forth in the License.
 *
 * See the NOTICE file distributed with this work for information regarding copyright ownership.
 */

package alluxio.server.ft;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThrows;

import alluxio.AlluxioURI;
import alluxio.AuthenticatedUserRule;
import alluxio.Constants;
import alluxio.client.file.FileSystem;
import alluxio.conf.Configuration;
import alluxio.conf.PropertyKey;
import alluxio.exception.FileAlreadyCompletedException;
import alluxio.exception.FileAlreadyExistsException;
import alluxio.exception.FileDoesNotExistException;
import alluxio.grpc.CompleteFilePOptions;
import alluxio.grpc.CreateFilePOptions;
import alluxio.grpc.DeletePOptions;
import alluxio.grpc.FileSystemMasterCommonPOptions;
import alluxio.grpc.MountPOptions;
import alluxio.grpc.RenamePOptions;
import alluxio.master.MultiMasterLocalAlluxioCluster;
import alluxio.master.file.FileSystemMaster;
import alluxio.master.file.contexts.CompleteFileContext;
import alluxio.master.file.contexts.CreateDirectoryContext;
import alluxio.master.file.contexts.CreateFileContext;
import alluxio.master.file.contexts.DeleteContext;
import alluxio.master.file.contexts.RenameContext;
import alluxio.testutils.BaseIntegrationTest;
import alluxio.testutils.IntegrationTestUtils;
import alluxio.util.io.PathUtils;
import alluxio.wire.FileInfo;
import alluxio.wire.OperationId;

import com.google.common.collect.ImmutableMap;
import org.apache.commons.io.IOUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.rules.TestName;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;

import java.io.FileWriter;
import java.nio.charset.Charset;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collection;
import java.util.Map;
import java.util.UUID;

@RunWith(Parameterized.class)
public final class FileSystemMasterFaultToleranceIntegrationTest extends BaseIntegrationTest {
  private static final int CLUSTER_WAIT_TIMEOUT_MS = 120 * Constants.SECOND_MS;
  private static final String TEST_USER = "test";

  private MultiMasterLocalAlluxioCluster mMultiMasterLocalAlluxioCluster;

  @Rule
  public TemporaryFolder mFolder = new TemporaryFolder();

  @Rule
  public TestName mTestName = new TestName();

  @Rule
  public AuthenticatedUserRule mAuthenticatedUser =
      new AuthenticatedUserRule(TEST_USER, Configuration.global());

  @Parameterized.Parameters
  public static Collection<Object[]> data() {
    return Arrays.asList(new Object[][] {
        {new ImmutableMap.Builder<PropertyKey, Object>()
            .put(PropertyKey.MASTER_FILE_SYSTEM_MERGE_INODE_JOURNALS, false)
            .build()},
        {new ImmutableMap.Builder<PropertyKey, Object>()
            .put(PropertyKey.MASTER_FILE_SYSTEM_MERGE_INODE_JOURNALS, true)
            .build()},
    });
  }

  @Parameterized.Parameter
  public ImmutableMap<PropertyKey, Object> mParameterizedConfigMap;

  @Before
  public final void before() throws Exception {
    mMultiMasterLocalAlluxioCluster = new MultiMasterLocalAlluxioCluster(2, 1);
    mMultiMasterLocalAlluxioCluster.initConfiguration(
        IntegrationTestUtils.getTestName(getClass().getSimpleName(), mTestName.getMethodName()));
    Configuration.set(PropertyKey.USER_RPC_RETRY_MAX_DURATION, "60sec");
    Configuration.set(PropertyKey.USER_RPC_RETRY_MAX_SLEEP_MS, "1sec");
    Configuration.set(PropertyKey.NETWORK_CONNECTION_SERVER_SHUTDOWN_TIMEOUT, "30sec");
    Configuration.set(PropertyKey.MASTER_JOURNAL_TAILER_SHUTDOWN_QUIET_WAIT_TIME_MS, "0sec");
    Configuration.set(PropertyKey.SECURITY_LOGIN_USERNAME, TEST_USER);
    Configuration.set(PropertyKey.MASTER_FILE_SYSTEM_MERGE_INODE_JOURNALS, false);
    for (Map.Entry<PropertyKey, Object> entry : mParameterizedConfigMap.entrySet()) {
      Configuration.set(entry.getKey(), entry.getValue());
    }
    mMultiMasterLocalAlluxioCluster.start();
  }

  @After
  public final void after() throws Exception {
    mMultiMasterLocalAlluxioCluster.stop();
    Configuration.reloadProperties();
  }

  @Test
  public void syncMetadataUFSFailOver() throws Exception {
    String ufsPath = mFolder.newFolder().getAbsoluteFile().toString();
    String ufsUri = "file://" + ufsPath;
    MountPOptions options = MountPOptions.newBuilder().build();
    FileSystem client = mMultiMasterLocalAlluxioCluster.getClient();
    AlluxioURI mountPath = new AlluxioURI("/mnt1");
    client.mount(mountPath, new AlluxioURI(ufsUri), options);

    // create files outside alluxio
    String fileName = "someFile";
    String contents = "contents";
    for (int i = 0; i < 100; i++) {
      try (FileWriter fw = new FileWriter(Paths.get(PathUtils.concatPath(
          ufsPath, fileName + i)).toString())) {
        fw.write(contents + i);
      }
    }
    for (int i = 0; i < 100; i++) {
      // sync it with metadata sync
      assertEquals(contents + i, IOUtils.toString(client.openFile(
          mountPath.join(fileName + i)), Charset.defaultCharset()));
    }

    // Promote standby to be a leader and reset test state.
    mMultiMasterLocalAlluxioCluster.stopLeader();
    mMultiMasterLocalAlluxioCluster.waitForNewMaster(CLUSTER_WAIT_TIMEOUT_MS);
    mMultiMasterLocalAlluxioCluster.waitForWorkersRegistered(CLUSTER_WAIT_TIMEOUT_MS);

    // read the files again
    client = mMultiMasterLocalAlluxioCluster.getClient();
    for (int i = 0; i < 100; i++) {
      assertEquals(contents + i, IOUtils.toString(client.openFile(
          mountPath.join(fileName + i)), Charset.defaultCharset()));
    }
  }

  @Test
  public void partitionTolerantCreateFile() throws Exception {
    // Create paths for the test.
    AlluxioURI testPath1 = new AlluxioURI("/testPath1");
    // Create context1 with unique operation id.
    CreateFileContext context = CreateFileContext
        .mergeFrom(CreateFilePOptions.newBuilder().setCommonOptions(FileSystemMasterCommonPOptions
            .newBuilder().setOperationId(new OperationId(UUID.randomUUID()).toFsProto())));
    // Create context2 with unique operation id.
    CreateFileContext context2 = CreateFileContext
        .mergeFrom(CreateFilePOptions.newBuilder().setCommonOptions(FileSystemMasterCommonPOptions
            .newBuilder().setOperationId(new OperationId(UUID.randomUUID()).toFsProto())));

    // Run partition tolerance test on leading master.
    {
      // Acquire file-system-master of leading master.
      final FileSystemMaster leadingFsMaster = mMultiMasterLocalAlluxioCluster
          .getLocalAlluxioMaster().getMasterProcess().getMaster(FileSystemMaster.class);

      // Create the path the first time.
      leadingFsMaster.createFile(testPath1, context);
      // Create the path the second time with the same context.
      // It should just return successfully.
      FileInfo fileInfo = leadingFsMaster.createFile(testPath1, context);
      Assert.assertEquals(fileInfo.getFileId(), leadingFsMaster.getFileId(testPath1));

      // Create the file again with a different context.
      // It should fail with `FileAlreadyExistException`.
      assertThrows(FileAlreadyExistsException.class,
          () -> leadingFsMaster.createFile(testPath1, context2));
    }

    // Promote standby to be a leader and reset test state.
    mMultiMasterLocalAlluxioCluster.stopLeader();
    mMultiMasterLocalAlluxioCluster.waitForNewMaster(CLUSTER_WAIT_TIMEOUT_MS);
    mAuthenticatedUser.resetUser();

    // Run partition tolerance test on the *new* leading master.
    {
      // Acquire file-system-master of leading master.
      final FileSystemMaster leadingFsMaster = mMultiMasterLocalAlluxioCluster
          .getLocalAlluxioMaster().getMasterProcess().getMaster(FileSystemMaster.class);

      // Creating on the new leader with the original operation-id should succeed.
      FileInfo fileInfo = leadingFsMaster.createFile(testPath1, context);
      Assert.assertEquals(fileInfo.getFileId(), leadingFsMaster.getFileId(testPath1));

      // Creating on the new leader with a different operation-id should fail.
      assertThrows(FileAlreadyExistsException.class,
          () -> leadingFsMaster.createFile(testPath1, context2));
    }
  }

  @Test
  public void partitionTolerantCompleteFile() throws Exception {
    // Create paths for the test.
    AlluxioURI testPath1 = new AlluxioURI("/testPath1");
    // Create context1 with unique operation id.
    CompleteFileContext context = CompleteFileContext
        .mergeFrom(CompleteFilePOptions.newBuilder().setCommonOptions(FileSystemMasterCommonPOptions
            .newBuilder().setOperationId(new OperationId(UUID.randomUUID()).toFsProto())));
    // Create context2 with unique operation id.
    CompleteFileContext context2 = CompleteFileContext
        .mergeFrom(CompleteFilePOptions.newBuilder().setCommonOptions(FileSystemMasterCommonPOptions
            .newBuilder().setOperationId(new OperationId(UUID.randomUUID()).toFsProto())));

    // Run partition tolerance test on leading master.
    {
      // Acquire file-system-master of leading master.
      FileSystemMaster leadingFsMaster = mMultiMasterLocalAlluxioCluster.getLocalAlluxioMaster()
          .getMasterProcess().getMaster(FileSystemMaster.class);

      // Create the path to complete.
      leadingFsMaster.createFile(testPath1, CreateFileContext.defaults());

      // Complete the path the first time.
      leadingFsMaster.completeFile(testPath1, context);
      // Complete the path the second time with the same context.
      // It should just return successfully.
      leadingFsMaster.completeFile(testPath1, context);

      // Complete the file again with a different context.
      // It should fail with `FileAlreadyCompletedException`.
      assertThrows(FileAlreadyCompletedException.class,
          () -> leadingFsMaster.completeFile(testPath1, context2));
    }

    // Promote standby to be a leader and reset test state.
    mMultiMasterLocalAlluxioCluster.stopLeader();
    mMultiMasterLocalAlluxioCluster.waitForNewMaster(CLUSTER_WAIT_TIMEOUT_MS);
    mAuthenticatedUser.resetUser();

    // Run partition tolerance test on the *new* leading master.
    {
      // Acquire file-system-master of leading master.
      FileSystemMaster leadingFsMaster = mMultiMasterLocalAlluxioCluster.getLocalAlluxioMaster()
          .getMasterProcess().getMaster(FileSystemMaster.class);

      // Completing the file on the new leader with the original operation-id should succeed.
      leadingFsMaster.completeFile(testPath1, context);

      // Creating on the new leader with a different operation-id.
      // It should fail with `FileAlreadyCompletedException`.
      assertThrows(FileAlreadyCompletedException.class,
          () -> leadingFsMaster.completeFile(testPath1, context2));
    }
  }

  @Test
  public void partitionTolerantDeleteFile() throws Exception {
    // Create paths for the test.
    AlluxioURI testPath1 = new AlluxioURI("/testPath1");
    // Create context1 with unique operation id.
    DeleteContext context = DeleteContext
        .mergeFrom(DeletePOptions.newBuilder().setCommonOptions(FileSystemMasterCommonPOptions
            .newBuilder().setOperationId(new OperationId(UUID.randomUUID()).toFsProto())));
    // Create context2 with unique operation id.
    DeleteContext context2 = DeleteContext
        .mergeFrom(DeletePOptions.newBuilder().setCommonOptions(FileSystemMasterCommonPOptions
            .newBuilder().setOperationId(new OperationId(UUID.randomUUID()).toFsProto())));
    {
      // Acquire file-system-master of leading master.
      FileSystemMaster leadingFsMaster = mMultiMasterLocalAlluxioCluster.getLocalAlluxioMaster()
          .getMasterProcess().getMaster(FileSystemMaster.class);

      // Create the path to delete.
      leadingFsMaster.createFile(testPath1, CreateFileContext.defaults());
      leadingFsMaster.completeFile(testPath1, CompleteFileContext.defaults());

      // Delete the path the first time.
      leadingFsMaster.delete(testPath1, context);
      // Delete the path the second time with the same context.
      // It should just return successfully.
      leadingFsMaster.delete(testPath1, context);

      // Delete the path again with a different context.
      // It should fail with `FileDoesNotExistException`.
      assertThrows(FileDoesNotExistException.class,
          () -> leadingFsMaster.delete(testPath1, context2));
    }

    // Promote standby to be a leader and reset test state.
    mMultiMasterLocalAlluxioCluster.stopLeader();
    mMultiMasterLocalAlluxioCluster.waitForNewMaster(CLUSTER_WAIT_TIMEOUT_MS);
    mAuthenticatedUser.resetUser();

    // Run partition tolerance test on the *new* leading master.
    {
      // Acquire file-system-master of leading master.
      FileSystemMaster leadingFsMaster = mMultiMasterLocalAlluxioCluster.getLocalAlluxioMaster()
          .getMasterProcess().getMaster(FileSystemMaster.class);
      // Deleting the file on the new leader with the original operation-id should succeed.
      leadingFsMaster.delete(testPath1, context);

      // Deleting on the new leader with a different operation-id.
      // It should fail with `FileDoesNotExistException`.
      assertThrows(FileDoesNotExistException.class,
          () -> leadingFsMaster.delete(testPath1, context2));
    }
  }

  @Test
  public void partitionTolerantDeleteDirectory() throws Exception {
    // Create paths for the test.
    AlluxioURI testPath1 = new AlluxioURI("/testPath1");
    // Create context1 with unique operation id.
    DeleteContext context = DeleteContext
        .mergeFrom(DeletePOptions.newBuilder().setCommonOptions(FileSystemMasterCommonPOptions
            .newBuilder().setOperationId(new OperationId(UUID.randomUUID()).toFsProto())));
    // Create context2 with unique operation id.
    DeleteContext context2 = DeleteContext
        .mergeFrom(DeletePOptions.newBuilder().setCommonOptions(FileSystemMasterCommonPOptions
            .newBuilder().setOperationId(new OperationId(UUID.randomUUID()).toFsProto())));

    // Run partition tolerance test on leading master.
    {
      // Acquire file-system-master of leading master.
      FileSystemMaster leadingFsMaster = mMultiMasterLocalAlluxioCluster.getLocalAlluxioMaster()
          .getMasterProcess().getMaster(FileSystemMaster.class);

      // Create the path to delete.
      leadingFsMaster.createDirectory(testPath1, CreateDirectoryContext.defaults());

      // Delete the path the first time.
      leadingFsMaster.delete(testPath1, context);
      // Delete the path the second time with the same context.
      // It should just return successfully.
      leadingFsMaster.delete(testPath1, context);

      // Delete the path again with a different context.
      // It should fail with `FileDoesNotExistException`.
      assertThrows(FileDoesNotExistException.class,
          () -> leadingFsMaster.delete(testPath1, context2));
    }

    // Promote standby to be a leader and reset test state.
    mMultiMasterLocalAlluxioCluster.stopLeader();
    mMultiMasterLocalAlluxioCluster.waitForNewMaster(CLUSTER_WAIT_TIMEOUT_MS);
    mAuthenticatedUser.resetUser();

    // Run partition tolerance test on the *new* leading master.
    {
      // Acquire file-system-master of leading master.
      FileSystemMaster leadingFsMaster = mMultiMasterLocalAlluxioCluster.getLocalAlluxioMaster()
          .getMasterProcess().getMaster(FileSystemMaster.class);

      // Deleting the path on the new leader with the original operation-id should succeed.
      leadingFsMaster.delete(testPath1, context);

      // Deleting on the new leader with a different operation-id.
      // It should fail with `FileDoesNotExistException`.
      assertThrows(FileDoesNotExistException.class,
          () -> leadingFsMaster.delete(testPath1, context2));
    }
  }

  @Test
  public void partitionTolerantRename() throws Exception {
    // Create paths for the test.
    AlluxioURI testPath1 = new AlluxioURI("/testPath1");
    AlluxioURI testPath2 = new AlluxioURI("/testPath2");
    // Create context1 with unique operation id.
    RenameContext context = RenameContext
        .mergeFrom(RenamePOptions.newBuilder().setCommonOptions(FileSystemMasterCommonPOptions
            .newBuilder().setOperationId(new OperationId(UUID.randomUUID()).toFsProto())));
    // Create context2 with unique operation id.
    RenameContext context2 = RenameContext
        .mergeFrom(RenamePOptions.newBuilder().setCommonOptions(FileSystemMasterCommonPOptions
            .newBuilder().setOperationId(new OperationId(UUID.randomUUID()).toFsProto())));

    // Run partition tolerance test on leading master.
    {
      // Acquire file-system-master of leading master.
      FileSystemMaster leadingFsMaster = mMultiMasterLocalAlluxioCluster.getLocalAlluxioMaster()
          .getMasterProcess().getMaster(FileSystemMaster.class);

      // Create the path to rename.
      leadingFsMaster.createDirectory(testPath1, CreateDirectoryContext.defaults());

      // Rename the path the first time.
      leadingFsMaster.rename(testPath1, testPath2, context);
      // Renaming the path the second time with the same context.
      // It should just return successfully.
      leadingFsMaster.rename(testPath1, testPath2, context);

      // Rename the path again with a different context.
      // It should fail with `FileDoesNotExistException`.
      assertThrows(FileDoesNotExistException.class,
          () -> leadingFsMaster.rename(testPath1, testPath2, context2));
    }

    // Promote standby to be a leader and reset test state.
    mMultiMasterLocalAlluxioCluster.stopLeader();
    mMultiMasterLocalAlluxioCluster.waitForNewMaster(CLUSTER_WAIT_TIMEOUT_MS);
    mAuthenticatedUser.resetUser();

    // Run partition tolerance test on the *new* leading master.
    {
      // Acquire file-system-master of leading master.
      FileSystemMaster leadingFsMaster = mMultiMasterLocalAlluxioCluster.getLocalAlluxioMaster()
          .getMasterProcess().getMaster(FileSystemMaster.class);

      // Renaming the path on the new leader with the original operation-id should succeed.
      leadingFsMaster.rename(testPath1, testPath2, context);

      // Renaming on the new leader with a different operation-id.
      // It should fail with `FileDoesNotExistException`.
      assertThrows(FileDoesNotExistException.class,
          () -> leadingFsMaster.rename(testPath1, testPath2, context2));
    }
  }
}
