alluxio-master-2.x/tests/src/test/java/alluxio/client/fs/FreeAndDeleteIntegrationTest.java
/*
 * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
 * (the "License"). You may not use this work except in compliance with the License, which is
 * available at www.apache.org/licenses/LICENSE-2.0
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied, as more fully set forth in the License.
 *
 * See the NOTICE file distributed with this work for information regarding copyright ownership.
 */

package alluxio.client.fs;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import alluxio.AlluxioURI;
import alluxio.client.file.FileOutStream;
import alluxio.client.file.FileSystem;
import alluxio.client.file.URIStatus;
import alluxio.conf.PropertyKey;
import alluxio.exception.FileDoesNotExistException;
import alluxio.grpc.CreateFilePOptions;
import alluxio.grpc.DeletePOptions;
import alluxio.grpc.WritePType;
import alluxio.master.block.BlockMaster;
import alluxio.master.file.meta.PersistenceState;
import alluxio.testutils.BaseIntegrationTest;
import alluxio.testutils.LocalAlluxioClusterResource;
import alluxio.util.CommonUtils;
import alluxio.util.WaitForOptions;
import alluxio.util.io.PathUtils;
import alluxio.wire.BlockInfo;
import alluxio.worker.block.BlockWorker;
import alluxio.worker.block.DefaultBlockWorker;

import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;

/**
 * Integration tests for file free and delete with under storage persisted.
 *
 */
public final class FreeAndDeleteIntegrationTest extends BaseIntegrationTest {
  private static final int USER_QUOTA_UNIT_BYTES = 1000;
  private static final int LOCK_POOL_LOW_WATERMARK = 50;
  private static final int LOCK_POOL_HIGH_WATERMARK = 100;
  private static final WaitForOptions WAIT_OPTIONS =
      WaitForOptions.defaults().setTimeoutMs(2000).setInterval(10);

  @Rule
  public LocalAlluxioClusterResource mLocalAlluxioClusterResource =
      new LocalAlluxioClusterResource.Builder()
          .setProperty(PropertyKey.USER_FILE_BUFFER_BYTES, USER_QUOTA_UNIT_BYTES)
          .setProperty(PropertyKey.MASTER_LOCK_POOL_LOW_WATERMARK, LOCK_POOL_LOW_WATERMARK)
          .setProperty(PropertyKey.MASTER_LOCK_POOL_HIGH_WATERMARK, LOCK_POOL_HIGH_WATERMARK)
          .setProperty(PropertyKey.MASTER_LOST_WORKER_FILE_DETECTION_INTERVAL, "10h")
          .build();

  private FileSystem mFileSystem = null;
  private CreateFilePOptions mWriteBoth;

  @Before
  public final void before() throws Exception {
    mFileSystem = mLocalAlluxioClusterResource.get().getClient();
    mWriteBoth = CreateFilePOptions.newBuilder().setWriteType(WritePType.CACHE_THROUGH)
        .setRecursive(true).build();
  }

  @Test
  public void freeAndDeleteIntegration() throws Exception {
    AlluxioURI filePath = new AlluxioURI(PathUtils.uniqPath());
    FileOutStream os = mFileSystem.createFile(filePath, mWriteBoth);
    os.write((byte) 0);
    os.write((byte) 1);
    os.close();

    URIStatus status = mFileSystem.getStatus(filePath);
    assertEquals(PersistenceState.PERSISTED.toString(), status.getPersistenceState());

    final Long blockId = status.getBlockIds().get(0);
    BlockMaster bm = mLocalAlluxioClusterResource.get().getLocalAlluxioMaster().getMasterProcess()
        .getMaster(BlockMaster.class);
    BlockInfo blockInfo = bm.getBlockInfo(blockId);
    assertEquals(2, blockInfo.getLength());
    assertFalse(blockInfo.getLocations().isEmpty());

    final DefaultBlockWorker bw = (DefaultBlockWorker)
        mLocalAlluxioClusterResource.get().getWorkerProcess().getWorker(BlockWorker.class);
    assertTrue(bw.getBlockStore().hasBlockMeta(blockId));
    assertEquals(0, bm.getLostBlocksCount());

    mFileSystem.free(filePath);

    CommonUtils.waitFor("file is freed", () -> {
      try {
        return 0 == mFileSystem.getStatus(filePath).getInAlluxioPercentage();
      } catch (Exception e) {
        return false;
      }
    }, WAIT_OPTIONS);

    status = mFileSystem.getStatus(filePath);
    // Verify block metadata in master is still present after block freed.
    assertEquals(1, status.getBlockIds().size());
    blockInfo = bm.getBlockInfo(status.getBlockIds().get(0));
    assertEquals(2, blockInfo.getLength());
    // Verify the block has been removed from all workers.
    assertTrue(blockInfo.getLocations().isEmpty());
    assertFalse(bw.getBlockStore().hasBlockMeta(blockId));
    // Verify the removed block is added to LostBlocks list.
    assertTrue(bm.isBlockLost(blockInfo.getBlockId()));

    mFileSystem.delete(filePath);

    try {
      // File is immediately gone after delete.
      mFileSystem.getStatus(filePath);
      Assert.fail(String.format("Expected file %s being deleted but it was not.", filePath));
    } catch (FileDoesNotExistException e) {
      // expected
    }

    // Verify the blocks are not in mLostBlocks.
    CommonUtils.waitFor("block is removed from mLostBlocks", () -> {
      try {
        return 0 == bm.getLostBlocksCount();
      } catch (Exception e) {
        return false;
      }
    }, WAIT_OPTIONS);
  }

  /**
   * Tests that deleting a directory with number of files larger than maximum lock cache size will
   * not be blocked.
   */
  @Test(timeout = 10000)
  public void deleteDir() throws Exception {
    String uniqPath = PathUtils.uniqPath();
    for (int file = 0; file < 2 * LOCK_POOL_HIGH_WATERMARK; file++) {
      AlluxioURI filePath = new AlluxioURI(PathUtils.concatPath(uniqPath, "file_" + file));
      mFileSystem.createFile(filePath, CreateFilePOptions.newBuilder().setRecursive(true).build())
          .close();
    }
    mFileSystem.delete(new AlluxioURI(uniqPath),
        DeletePOptions.newBuilder().setRecursive(true).build());
  }
}
