alluxio-master-2.x/core/common/src/main/java/alluxio/underfs/BaseUnderFileSystem.java
/*
 * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
 * (the "License"). You may not use this work except in compliance with the License, which is
 * available at www.apache.org/licenses/LICENSE-2.0
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied, as more fully set forth in the License.
 *
 * See the NOTICE file distributed with this work for information regarding copyright ownership.
 */

package alluxio.underfs;

import alluxio.AlluxioURI;
import alluxio.Constants;
import alluxio.SyncInfo;
import alluxio.collections.Pair;
import alluxio.conf.AlluxioConfiguration;
import alluxio.conf.PropertyKey;
import alluxio.file.options.DescendantType;
import alluxio.security.authorization.AccessControlList;
import alluxio.security.authorization.AclEntry;
import alluxio.security.authorization.DefaultAccessControlList;
import alluxio.underfs.options.CreateOptions;
import alluxio.underfs.options.DeleteOptions;
import alluxio.underfs.options.ListOptions;
import alluxio.underfs.options.MkdirsOptions;
import alluxio.underfs.options.OpenOptions;
import alluxio.util.RateLimiter;
import alluxio.util.ThreadFactoryUtils;
import alluxio.util.io.PathUtils;

import com.google.common.base.Preconditions;
import com.google.common.collect.Iterators;
import com.google.common.io.Closer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Consumer;
import java.util.stream.Stream;
import javax.annotation.Nullable;
import javax.annotation.concurrent.ThreadSafe;

/**
 * A base abstract {@link UnderFileSystem}.
 */
@ThreadSafe
public abstract class BaseUnderFileSystem implements UnderFileSystem, UfsClient {
  private static final Logger LOG = LoggerFactory.getLogger(BaseUnderFileSystem.class);
  public static final Pair<AccessControlList, DefaultAccessControlList> EMPTY_ACL =
      new Pair<>(null, null);

  /** The UFS {@link AlluxioURI} used to create this {@link BaseUnderFileSystem}. */
  protected final AlluxioURI mUri;

  /** UFS Configuration options. */
  protected final UnderFileSystemConfiguration mUfsConf;

  private final ExecutorService mAsyncIOExecutor;

  private final RateLimiter mRateLimiter;

  /**
   * Constructs an {@link BaseUnderFileSystem}.
   *
   * @param uri the {@link AlluxioURI} used to create this ufs
   * @param ufsConf UFS configuration
   */
  protected BaseUnderFileSystem(AlluxioURI uri, UnderFileSystemConfiguration ufsConf) {
    mUri = Preconditions.checkNotNull(uri, "uri");
    mUfsConf = Preconditions.checkNotNull(ufsConf, "ufsConf");
    mAsyncIOExecutor = Executors.newCachedThreadPool(
        ThreadFactoryUtils.build(uri.getPath() + "IOThread", true));
    long rateLimit = mUfsConf.isSet(PropertyKey.MASTER_METADATA_SYNC_UFS_RATE_LIMIT)
        ? mUfsConf.getLong(PropertyKey.MASTER_METADATA_SYNC_UFS_RATE_LIMIT) : 0;
    mRateLimiter = RateLimiter.createRateLimiter(rateLimit);
  }

  @Override
  public void close() throws IOException {
    try (Closer closer = Closer.create()) {
      closer.register(() -> {
        if (mAsyncIOExecutor != null) {
          mAsyncIOExecutor.shutdown();
        }
      });
    }
  }

  @Override
  public RateLimiter getRateLimiter() {
    return mRateLimiter;
  }

  @Override
  public OutputStream create(String path) throws IOException {
    return create(path, CreateOptions.defaults(mUfsConf).setCreateParent(true));
  }

  @Override
  public boolean deleteDirectory(String path) throws IOException {
    return deleteDirectory(path, DeleteOptions.defaults());
  }

  @Override
  public boolean exists(String path) throws IOException {
    return isFile(path) || isDirectory(path);
  }

  @Override
  public Pair<AccessControlList, DefaultAccessControlList> getAclPair(String path)
      throws IOException {
    return EMPTY_ACL;
  }

  @Override
  public void setAclEntries(String path, List<AclEntry> aclEntries) throws IOException {
    // Noop here by default
  }

  @Override
  public AlluxioConfiguration getConfiguration() {
    return mUfsConf;
  }

  @Override
  public String getFingerprint(String path) {
    // TODO(yuzhu): include default ACL in the fingerprint
    try {
      UfsStatus status = getStatus(path);
      Pair<AccessControlList, DefaultAccessControlList> aclPair = getAclPair(path);

      if (aclPair == null || aclPair.getFirst() == null || !aclPair.getFirst().hasExtended()) {
        return Fingerprint.create(getUnderFSType(), status).serialize();
      } else {
        return Fingerprint.create(getUnderFSType(), status, null, aclPair.getFirst()).serialize();
      }
    } catch (Exception e) {
      // In certain scenarios, it is expected that the UFS path does not exist.
      LOG.debug("Failed fingerprint. path: {} error: {}", path, e.toString());
      return Constants.INVALID_UFS_FINGERPRINT;
    }
  }

  @Override
  public Fingerprint getParsedFingerprint(String path) {
    return getParsedFingerprint(path, null);
  }

  @Override
  public Fingerprint getParsedFingerprint(String path, @Nullable String contentHash) {
    try {
      UfsStatus status = getStatus(path);
      Pair<AccessControlList, DefaultAccessControlList> aclPair = getAclPair(path);

      if (aclPair == null || aclPair.getFirst() == null || !aclPair.getFirst().hasExtended()) {
        return Fingerprint.create(getUnderFSType(), status, contentHash);
      } else {
        return Fingerprint.create(getUnderFSType(), status, contentHash, aclPair.getFirst());
      }
    } catch (IOException e) {
      return Fingerprint.INVALID_FINGERPRINT;
    }
  }

  @Override
  public UfsMode getOperationMode(Map<String, UfsMode> physicalUfsState) {
    UfsMode ufsMode = physicalUfsState.get(mUri.getRootPath());
    if (ufsMode != null) {
      return ufsMode;
    }
    return UfsMode.READ_WRITE;
  }

  @Override
  public List<String> getPhysicalStores() {
    return new ArrayList<>(Arrays.asList(mUri.getRootPath()));
  }

  @Override
  public boolean isObjectStorage() {
    return false;
  }

  @Override
  public boolean isSeekable() {
    return false;
  }

  @Nullable
  @Override
  public Iterator<UfsStatus> listStatusIterable(
      String path, ListOptions options, String startAfter, int batchSize) throws IOException {
    // Calling this method on non s3 UFS might result in OOM because batch based fetching
    // is not supported and this method essentially fetches all ufs status and converts it to
    // an iterator.
    UfsStatus[] result = listStatus(path, options);
    if (result == null) {
      return null;
    }
    Arrays.sort(result, Comparator.comparing(UfsStatus::getName));
    return Iterators.forArray(result);
  }

  @Override
  public void performListingAsync(
      String path, @Nullable String continuationToken, @Nullable String startAfter,
      DescendantType descendantType, boolean checkStatus, Consumer<UfsLoadResult> onComplete,
      Consumer<Throwable> onError) {
    mAsyncIOExecutor.submit(() -> {
      try {
        UfsStatus baseStatus = null;
        if (checkStatus) {
          try {
            baseStatus = getStatus(path);
            if (baseStatus == null && !isObjectStorage()) {
              onComplete.accept(new UfsLoadResult(Stream.empty(), 0,
                  null, null, false, false, false));
              return;
            }
            if (baseStatus != null && (descendantType == DescendantType.NONE
                || baseStatus.isFile())) {
              onComplete.accept(new UfsLoadResult(Stream.of(baseStatus), 1,
                  null, new AlluxioURI(baseStatus.getName()), false,
                  baseStatus.isFile(), isObjectStorage()));
              return;
            }
          } catch (FileNotFoundException e) {
            // if we are not using object storage we know nothing exists at the path,
            // so just return an empty result
            if (!isObjectStorage()) {
              onComplete.accept(new UfsLoadResult(Stream.empty(), 0,
                  null, null, false, false, false));
              return;
            }
          }
        }
        UfsStatus[] items = listStatus(path, ListOptions.defaults()
            .setRecursive(descendantType == DescendantType.ALL));
        if (items != null) {
          if (descendantType == DescendantType.NONE && items.length > 0) {
            assert isObjectStorage() && this instanceof ObjectUnderFileSystem;
            ObjectUnderFileSystem.ObjectPermissions permissions =
                ((ObjectUnderFileSystem) this).getPermissions();
            items = new UfsStatus[] {
                new UfsDirectoryStatus("", permissions.getOwner(), permissions.getGroup(),
                    permissions.getMode())};
          }
          Arrays.sort(items, Comparator.comparing(UfsStatus::getName));
          for (UfsStatus item: items) {
            // performListingAsync is used by metadata sync v2
            // which expects the name of an item to be a full path
            item.setName(PathUtils.concatPath(path, item.getName()));
          }
        }
        if (items != null && items.length == 0) {
          items = null;
        }
        UfsStatus firstItem = baseStatus != null ? baseStatus
            : items != null ? items[0] : null;
        UfsStatus lastItem = items == null ? firstItem
            : items[items.length - 1];
        Stream<UfsStatus> itemStream = items == null ? Stream.empty() : Arrays.stream(items);
        int itemCount = items == null ? 0 : items.length;
        if (baseStatus != null) {
          itemStream = Stream.concat(Stream.of(baseStatus), itemStream);
          itemCount++;
        }
        onComplete.accept(new UfsLoadResult(itemStream, itemCount,
            null, lastItem == null ? null : new AlluxioURI(lastItem.getName()), false,
            firstItem != null && firstItem.isFile(), isObjectStorage()));
      } catch (Throwable t) {
        onError.accept(t);
      }
    });
  }

  @Override
  @Nullable
  public UfsStatus[] listStatus(String path, ListOptions options) throws IOException {
    if (!options.isRecursive()) {
      return listStatus(path);
    }
    path = validatePath(path);
    List<UfsStatus> returnPaths = new ArrayList<>();
    // Each element is a pair of (full path, UfsStatus)
    Queue<Pair<String, UfsStatus>> pathsToProcess = new ArrayDeque<>();
    // We call list initially, so we can return null if the path doesn't denote a directory
    UfsStatus[] statuses = listStatus(path);
    if (statuses == null) {
      return null;
    } else {
      for (UfsStatus status : statuses) {
        pathsToProcess.add(new Pair<>(PathUtils.concatPath(path, status.getName()), status));
      }
    }
    while (!pathsToProcess.isEmpty()) {
      final Pair<String, UfsStatus> pathToProcessPair = pathsToProcess.remove();
      final String pathToProcess = pathToProcessPair.getFirst();
      UfsStatus pathStatus = pathToProcessPair.getSecond();
      int beginIndex = path.endsWith(AlluxioURI.SEPARATOR) ? path.length() : path.length() + 1;
      returnPaths.add(pathStatus.setName(pathToProcess.substring(beginIndex)));

      if (pathStatus.isDirectory()) {
        // Add all of its subpaths
        UfsStatus[] children = listStatus(pathToProcess);
        if (children != null) {
          for (UfsStatus child : children) {
            pathsToProcess.add(
                new Pair<>(PathUtils.concatPath(pathToProcess, child.getName()), child));
          }
        }
      }
    }
    return returnPaths.toArray(new UfsStatus[0]);
  }

  @Override
  public InputStream open(String path) throws IOException {
    return open(path, OpenOptions.defaults());
  }

  @Override
  public boolean mkdirs(String path) throws IOException {
    return mkdirs(path, MkdirsOptions.defaults(mUfsConf));
  }

  @Override
  public AlluxioURI resolveUri(AlluxioURI ufsBaseUri, String alluxioPath) {
    return new AlluxioURI(ufsBaseUri, PathUtils.concatPath(ufsBaseUri.getPath(), alluxioPath),
        false);
  }

  @Override
  public boolean supportsActiveSync() {
    return false;
  }

  @Override
  public SyncInfo getActiveSyncInfo() {
    return SyncInfo.emptyInfo();
  }

  @Override
  public boolean startActiveSyncPolling(long txId) throws IOException {
    return false;
  }

  @Override
  public boolean stopActiveSyncPolling() {
    return false;
  }

  @Override
  public void startSync(AlluxioURI uri)  { }

  @Override
  public void stopSync(AlluxioURI uri) { }

  /**
   * Clean the path by creating a URI and turning it back to a string.
   *
   * @param path the path to validate
   * @return validated path
   */
  protected static String validatePath(String path) {
    return new AlluxioURI(path).toString();
  }
}
