alluxio-master-2.x/core/common/src/main/java/alluxio/util/io/BufferUtils.java
/*
 * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
 * (the "License"). You may not use this work except in compliance with the License, which is
 * available at www.apache.org/licenses/LICENSE-2.0
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied, as more fully set forth in the License.
 *
 * See the NOTICE file distributed with this work for information regarding copyright ownership.
 */

package alluxio.util.io;

import alluxio.Constants;
import alluxio.util.CommonUtils;

import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.FileOutputStream;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.util.ArrayList;
import java.util.List;
import javax.annotation.concurrent.ThreadSafe;

/**
 * Utilities related to buffers, not only {@link ByteBuffer}.
 */
@ThreadSafe
public final class BufferUtils {
  private static final Logger LOG = LoggerFactory.getLogger(BufferUtils.class);
  private static final Object LOCK = new Object();
  private static final int TRANSFER_BUFFER_SIZE = 4 * Constants.MB;
  private static volatile Method sCleanerCleanMethod;
  private static volatile Method sByteBufferCleanerMethod;
  private static volatile Class sUnsafeClass;

  /**
   * Converts a byte to an integer.
   *
   * @param b the byte to convert
   * @return the integer representation of the byte
   */
  public static int byteToInt(byte b) {
    return b & 0xFF;
  }

  /**
   * Forces to unmap a direct buffer if this buffer is no longer used. After calling this method,
   * this direct buffer should be discarded. This is unsafe operation and currently a work-around to
   * avoid huge memory occupation caused by memory map.
   *
   * @param buffer bytebuffer
   */
  public static void cleanDirectBuffer(ByteBuffer buffer) {
    Preconditions.checkNotNull(buffer, "buffer is null");
    Preconditions.checkArgument(buffer.isDirect(), "buffer isn't a DirectByteBuffer");
    int javaVersion = CommonUtils.getJavaVersion();
    if (javaVersion < 9) {
      cleanDirectBufferJava8(buffer);
    } else {
      cleanDirectBufferJava11(buffer);
    }
  }

  /**
   * <p>
   * Note: This calls the cleaner method on jdk 9+.
   * See <a
   * href="https://stackoverflow.com/questions/2972986/how-to-unmap-a-file-from-memory-mapped-
   * using-filechannel-in-java"
   * >more discussion</a>.
   *
   * @param buffer the byte buffer to be unmapped, this must be a direct buffer
   */
  private static void cleanDirectBufferJava11(ByteBuffer buffer) {
    if (sByteBufferCleanerMethod == null || sUnsafeClass == null) {
      synchronized (LOCK) {
        try {
          if (sByteBufferCleanerMethod == null || sUnsafeClass == null) {
            try {
              sUnsafeClass = Class.forName("sun.misc.Unsafe");
            } catch (Exception e) {
              // jdk.internal.misc.Unsafe doesn't yet have an invokeCleaner() method,
              // but that method should be added if sun.misc.Unsafe is removed.
              sUnsafeClass = Class.forName("jdk.internal.misc.Unsafe");
            }
            sByteBufferCleanerMethod = sUnsafeClass.getMethod("invokeCleaner", ByteBuffer.class);
            sByteBufferCleanerMethod.setAccessible(true);
          }
        } catch (Exception e) {
          // Force to drop reference to the buffer to clean
          buffer = null;
          return;
        }
      }
    }
    try {
      Field theUnsafeField = sUnsafeClass.getDeclaredField("theUnsafe");
      theUnsafeField.setAccessible(true);
      Object theUnsafe = theUnsafeField.get(null);
      sByteBufferCleanerMethod.invoke(theUnsafe, buffer);
    } catch (Exception e) {
      LOG.warn("Failed to unmap direct ByteBuffer: {}, error message: {}",
          buffer.getClass().getName(), e.toString());
    }
  }

  /**
   * <p>
   * NOTE: DirectByteBuffers are not guaranteed to be garbage-collected immediately after their
   * references are released and may lead to OutOfMemoryError. This function helps by calling the
   * Cleaner method of a DirectByteBuffer explicitly. See <a
   * href="http://stackoverflow.com/questions/1854398/how-to-garbage-collect-a-direct-buffer-java"
   * >more discussion</a>.
   *
   * @param buffer the byte buffer to be unmapped, this must be a direct buffer
   */
  private static void cleanDirectBufferJava8(ByteBuffer buffer) {
    if (sByteBufferCleanerMethod == null) {
      synchronized (LOCK) {
        try {
          if (sByteBufferCleanerMethod == null) {
            sByteBufferCleanerMethod = buffer.getClass().getMethod("cleaner");
            sByteBufferCleanerMethod.setAccessible(true);
          }
        } catch (Exception e) {
          // Force to drop reference to the buffer to clean
          buffer = null;
          return;
        }
      }
    }
    try {
      final Object cleaner = sByteBufferCleanerMethod.invoke(buffer);
      if (cleaner == null) {
        if (buffer.capacity() > 0) {
          LOG.warn("Failed to get cleaner for ByteBuffer: {}", buffer.getClass().getName());
        }
        // The cleaner could be null when the buffer is initialized as zero capacity.
        return;
      }
      if (sCleanerCleanMethod == null) {
        synchronized (LOCK) {
          if (sCleanerCleanMethod == null) {
            sCleanerCleanMethod = cleaner.getClass().getMethod("clean");
          }
        }
      }
      sCleanerCleanMethod.invoke(cleaner);
    } catch (Exception e) {
      LOG.warn("Failed to unmap direct ByteBuffer: {}, error message: {}",
          buffer.getClass().getName(), e.toString());
    } finally {
      // Force to drop reference to the buffer to clean
      buffer = null;
    }
  }

  /**
   * Clones a {@link ByteBuffer}.
   * <p>
   * The new bytebuffer will have the same content, but the type of the bytebuffer may not be the
   * same.
   *
   * @param buf The ByteBuffer to copy
   * @return The new ByteBuffer
   */
  public static ByteBuffer cloneByteBuffer(ByteBuffer buf) {
    ByteBuffer ret = ByteBuffer.allocate(buf.limit() - buf.position());
    if (buf.hasArray()) {
      ret.put(buf.array(), buf.position(), buf.limit() - buf.position());
    } else {
      // direct buffer
      ret.put(buf);
    }
    ret.flip();
    return ret;
  }

  /**
   * Clones a list of {@link ByteBuffer}s.
   *
   * @param source the list of ByteBuffers to copy
   * @return the new list of ByteBuffers
   */
  public static List<ByteBuffer> cloneByteBufferList(List<ByteBuffer> source) {
    List<ByteBuffer> ret = new ArrayList<>(source.size());
    for (ByteBuffer b : source) {
      ret.add(cloneByteBuffer(b));
    }
    return ret;
  }

  /**
   * Puts a byte (the first byte of an integer) into a {@link ByteBuffer}.
   *
   * @param buf ByteBuffer to use
   * @param b byte to put into the ByteBuffer
   */
  public static void putIntByteBuffer(ByteBuffer buf, int b) {
    buf.put((byte) (b & 0xFF));
  }

  /**
   * Gets an increasing sequence of bytes starting at zero.
   *
   * @param len the target length of the sequence
   * @return an increasing sequence of bytes
   */
  public static byte[] getIncreasingByteArray(int len) {
    return getIncreasingByteArray(0, len);
  }

  /**
   * Gets an increasing sequence of bytes starting with the given value.
   *
   * @param start the starting value to use
   * @param len the target length of the sequence
   * @return an increasing sequence of bytes
   */
  public static byte[] getIncreasingByteArray(int start, int len) {
    byte[] ret = new byte[len];
    for (int k = 0; k < len; k++) {
      ret[k] = (byte) (k + start);
    }
    return ret;
  }

  /**
   * Checks if the given byte array starts with a constant sequence of bytes of the given value
   * and length.
   *
   * @param value the value to check for
   * @param len the target length of the sequence
   * @param arr the byte array to check
   * @return true if the byte array has a prefix of length {@code len} that is a constant
   *         sequence of bytes of the given value
   */
  public static boolean equalConstantByteArray(byte value, int len, byte[] arr) {
    if (arr == null || arr.length != len) {
      return false;
    }
    for (int k = 0; k < len; k++) {
      if (arr[k] != value) {
        return false;
      }
    }
    return true;
  }

  /**
   * Checks if the given byte array starts with an increasing sequence of bytes of the given
   * length, starting from 0.
   *
   * @param len the target length of the sequence
   * @param arr the byte array to check
   * @return true if the byte array has a prefix of length {@code len} that is an increasing
   *         sequence of bytes starting at zero
   */
  public static boolean equalIncreasingByteArray(int len, byte[] arr) {
    return equalIncreasingByteArray(0, len, arr);
  }

  /**
   * Checks if the given byte array starts with an increasing sequence of bytes of the given
   * length, starting from the given value. The array length must be equal to the length checked.
   *
   * @param start the starting value to use
   * @param len the target length of the sequence
   * @param arr the byte array to check
   * @return true if the byte array has a prefix of length {@code len} that is an increasing
   *         sequence of bytes starting at {@code start}
   */
  public static boolean equalIncreasingByteArray(int start, int len, byte[] arr) {
    if (arr == null || arr.length != len) {
      return false;
    }
    for (int k = 0; k < len; k++) {
      if (arr[k] != (byte) (start + k)) {
        return false;
      }
    }
    return true;
  }

  /**
   * Gets a {@link ByteBuffer} containing an increasing sequence of bytes starting at zero.
   *
   * @param len the target length of the sequence
   * @return ByteBuffer containing an increasing sequence of bytes
   */
  public static ByteBuffer getIncreasingByteBuffer(int len) {
    return getIncreasingByteBuffer(0, len);
  }

  /**
   * Gets a {@link ByteBuffer} containing an increasing sequence of bytes starting at the given
   * value.
   *
   * @param len the target length of the sequence
   * @param start the starting value to use
   * @return ByteBuffer containing an increasing sequence of bytes
   */
  public static ByteBuffer getIncreasingByteBuffer(int start, int len) {
    return ByteBuffer.wrap(getIncreasingByteArray(start, len));
  }

  /**
   * Checks if the given {@link ByteBuffer} starts with an increasing sequence of bytes starting at
   * the given value of length equal to or greater than the given length.
   *
   * @param start the starting value to use
   * @param len the target length of the sequence
   * @param buf the ByteBuffer to check
   * @return true if the ByteBuffer has a prefix of length {@code len} that is an increasing
   *         sequence of bytes starting at {@code start}
   */
  public static boolean equalIncreasingByteBuffer(int start, int len, ByteBuffer buf) {
    if (buf == null) {
      return false;
    }
    buf.rewind();
    if (buf.remaining() != len) {
      return false;
    }
    for (int k = 0; k < len; k++) {
      if (buf.get() != (byte) (start + k)) {
        return false;
      }
    }
    return true;
  }

  /**
   * Checks if the given byte array has a sub array equals an increasing sequence of bytes,
   * starting from 0.
   *
   * @param startIndex the start index in the array to check against
   * @param endIndex the end index in the array to check against
   * @param arr the byte array to check
   * @return true if match, false otherwise
   */
  public static boolean matchIncreasingByteArray(int startIndex, int endIndex, byte[] arr) {
    return matchIncreasingByteArray(0, startIndex, endIndex, arr);
  }

  /**
   * Checks if the given byte array has a sub array equals an increasing sequence of bytes,
   * starting from the given value.
   *
   * @param startNum the starting value to use
   * @param startIndex the start index in the array to check against
   * @param endIndex the end index in the array to check against
   * @param arr the byte array to check
   * @return true if match, false otherwise
   */
  public static boolean matchIncreasingByteArray(
      int startNum, int startIndex, int endIndex, byte[] arr) {
    if (arr == null) {
      return false;
    }
    for (int k = startIndex; k < endIndex; k++) {
      if (arr[k] != (byte) (startNum + k)) {
        return false;
      }
    }
    return true;
  }

  /**
   * Checks if the given byte buffer has a sub buffer equals an increasing sequence of bytes,
   * starting from the given value.
   *
   * @param startIndex the start index in the array to check against
   * @param endIndex the end index in the array to check against
   * @param buf the byte buffer to check
   * @return true if match, false otherwise
   */
  public static boolean matchIncreasingByteBuffer(int startIndex, int endIndex, ByteBuffer buf) {
    return matchIncreasingByteBuffer(0, startIndex, endIndex, buf);
  }

  /**
   * Checks if the given byte buffer has a sub buffer equals an increasing sequence of bytes,
   * starting from the given value.
   *
   * @param startNum the starting value to use
   * @param startIndex the start index in the array to check against
   * @param endIndex the end index in the array to check against
   * @param buf the byte buffer to check
   * @return true if match, false otherwise
   */
  public static boolean matchIncreasingByteBuffer(
      int startNum, int startIndex, int endIndex, ByteBuffer buf) {
    if (buf == null) {
      return false;
    }
    buf.rewind();
    buf.position(startIndex);
    for (int k = 0; k < endIndex - startIndex; k++) {
      if (buf.get() != (byte) (startNum + k)) {
        return false;
      }
    }
    return true;
  }

  /**
   * Writes buffer to the given file path.
   *
   * @param path file path to write the data
   * @param buffer raw data
   */
  public static void writeBufferToFile(String path, byte[] buffer) throws IOException {
    try (FileOutputStream os = new FileOutputStream(path)) {
      os.write(buffer);
    }
  }

  /**
   * An efficient copy between two channels with a fixed-size buffer.
   *
   * @param src the source channel
   * @param dest the destination channel
   */
  public static void transfer(final ReadableByteChannel src, final WritableByteChannel dest)
      throws IOException {
    final ByteBuffer buffer = ByteBuffer.allocateDirect(TRANSFER_BUFFER_SIZE);
    try {
      while (src.read(buffer) != -1) {
        buffer.flip();
        dest.write(buffer);
        buffer.compact();
      }
      buffer.flip();
      while (buffer.hasRemaining()) {
        dest.write(buffer);
      }
    } finally {
      cleanDirectBuffer(buffer);
    }
  }

  /**
   * Gets the unsigned byte value of an integer. Useful for comparing to the result of reading
   * from an input stream.
   * @param i the integer to convert
   * @return the integer value after casting as an unsigned byte
   */
  public static int intAsUnsignedByteValue(int i) {
    return ((byte) i) & 0xFF;
  }

  /**
   * Creates a byte array from the given ByteBuffer, the position property of input
   * {@link ByteBuffer} remains unchanged.
   *
   * @param buf source ByteBuffer
   * @return a newly created byte array
   */
  public static byte[] newByteArrayFromByteBuffer(ByteBuffer buf) {
    final int length = buf.remaining();
    byte[] bytes = new byte[length];
    // transfer bytes from this buffer into the given destination array
    buf.duplicate().get(bytes, 0, length);
    return bytes;
  }

  /**
   * Creates a new ByteBuffer sliced from a given ByteBuffer. The new ByteBuffer shares the
   * content of the existing one, but with independent position/mark/limit. After slicing, the
   * new ByteBuffer has position 0, and the input ByteBuffer is unmodified.
   *
   * @param buffer source ByteBuffer to slice
   * @param position position in the source ByteBuffer to slice
   * @param length length of the sliced ByteBuffer
   * @return the sliced ByteBuffer
   */
  public static ByteBuffer sliceByteBuffer(ByteBuffer buffer, int position, int length) {
    ByteBuffer slicedBuffer = ((ByteBuffer) buffer.duplicate().position(position)).slice();
    slicedBuffer.limit(length);
    return slicedBuffer;
  }

  /**
   * Convenience method for {@link #sliceByteBuffer(ByteBuffer, int, int)} where the last parameter
   * is the number of remaining bytes in the new buffer.
   *
   * @param buffer source {@link ByteBuffer} to slice
   * @param position position in the source {@link ByteBuffer} to slice
   * @return the sliced {@link ByteBuffer}
   */
  public static ByteBuffer sliceByteBuffer(ByteBuffer buffer, int position) {
    // The following is an optimization comparing to directly calling
    // sliceByteBuffer(ByteBuffer, int, int) needs to compute the length of the sliced buffer and
    // set the limit, but those operations should have been taken care of by the slice() method.
    return ((ByteBuffer) buffer.duplicate().position(position)).slice();
  }

  private BufferUtils() {} // prevent instantiation
}
