cassandra-trunk/src/java/org/apache/cassandra/io/util/FileUtils.java
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.cassandra.io.util;

import java.io.BufferedWriter;
import java.io.Closeable;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodHandles;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.ClosedChannelException;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryNotEmptyException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.text.DecimalFormat;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.RateLimiter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.cassandra.io.FSError;
import org.apache.cassandra.io.FSErrorHandler;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.SyncUtil;

import static org.apache.cassandra.config.CassandraRelevantProperties.JAVA_IO_TMPDIR;
import static org.apache.cassandra.utils.Throwables.maybeFail;

public final class FileUtils
{
    public static final Charset CHARSET = StandardCharsets.UTF_8;

    private static final Logger logger = LoggerFactory.getLogger(FileUtils.class);

    public static final long ONE_KIB = 1024;
    public static final long ONE_MIB = 1024 * ONE_KIB;
    public static final long ONE_GIB = 1024 * ONE_MIB;
    public static final long ONE_TIB = 1024 * ONE_GIB;

    private static final DecimalFormat df = new DecimalFormat("#.##");
    private static final AtomicReference<Optional<FSErrorHandler>> fsErrorHandler = new AtomicReference<>(Optional.empty());

    private static final Class clsDirectBuffer;
    private static final MethodHandle mhDirectBufferCleaner;
    private static final MethodHandle mhCleanerClean;

    static
    {
        try
        {
            clsDirectBuffer = Class.forName("sun.nio.ch.DirectBuffer");
            Method mDirectBufferCleaner = clsDirectBuffer.getMethod("cleaner");
            mhDirectBufferCleaner = MethodHandles.lookup().unreflect(mDirectBufferCleaner);
            Method mCleanerClean = mDirectBufferCleaner.getReturnType().getMethod("clean");
            mhCleanerClean = MethodHandles.lookup().unreflect(mCleanerClean);

            ByteBuffer buf = ByteBuffer.allocateDirect(1);
            clean(buf);
        }
        catch (IllegalAccessException e)
        {
            logger.error("FATAL: Cassandra is unable to access required classes. This usually means it has been " +
                "run without the aid of the standard startup scripts or the scripts have been edited. If this was " +
                "intentional, and you are attempting to use Java 11+ you may need to add the --add-exports and " +
                "--add-opens jvm options from either jvm11-server.options or jvm11-client.options", e);
            throw new RuntimeException(e);  // causes ExceptionInInitializerError, will prevent startup
        }
        catch (Throwable t)
        {
            logger.error("FATAL: Cannot initialize optimized memory deallocator.", t);
            JVMStabilityInspector.inspectThrowable(t);
            throw new RuntimeException(t); // causes ExceptionInInitializerError, will prevent startup
        }
    }

    private static final File tempDir = new File(JAVA_IO_TMPDIR.getString());
    private static final AtomicLong tempFileNum = new AtomicLong();

    public static File getTempDir()
    {
        return tempDir;
    }

    /**
     * Pretty much like {@link java.io.File#createTempFile(String, String, java.io.File)}, but with
     * the guarantee that the "random" part of the generated file name between
     * {@code prefix} and {@code suffix} is a positive, increasing {@code long} value.
     */
    public static File createTempFile(String prefix, String suffix, File directory)
    {
        // Do not use java.io.File.createTempFile(), because some tests rely on the
        // behavior that the "random" part in the temp file name is a positive 'long'.
        // However, at least since Java 9 the code to generate the "random" part
        // uses an _unsigned_ random long generated like this:
        // Long.toUnsignedString(new java.util.Random.nextLong())

        while (true)
        {
            // The contract of File.createTempFile() says, that it must not return
            // the same file name again. We do that here in a very simple way,
            // that probably doesn't cover all edge cases. Just rely on system
            // wall clock and return strictly increasing values from that.
            long num = tempFileNum.getAndIncrement();

            // We have a positive long here, which is safe to use for example
            // for CommitLogTest.
            String fileName = prefix + num + suffix;
            File candidate = new File(directory, fileName);
            if (candidate.createFileIfNotExists())
                return candidate;
        }
    }

    public static File createTempFile(String prefix, String suffix)
    {
        return createTempFile(prefix, suffix, tempDir);
    }

    public static File createDeletableTempFile(String prefix, String suffix)
    {
        File f = createTempFile(prefix, suffix, getTempDir());
        f.deleteOnExit();
        return f;
    }

    public static void createHardLink(String from, String to)
    {
        createHardLink(new File(from), new File(to));
    }

    public static void createHardLink(File from, File to)
    {
        if (to.exists())
            throw new RuntimeException("Tried to create duplicate hard link to " + to);
        if (!from.exists())
            throw new RuntimeException("Tried to hard link to file that does not exist " + from);

        try
        {
            Files.createLink(to.toPath(), from.toPath());
        }
        catch (IOException e)
        {
            throw new FSWriteError(e, to);
        }
    }

    public static void createHardLinkWithConfirm(String from, String to)
    {
        createHardLinkWithConfirm(new File(from), new File(to));
    }

    public static void createHardLinkWithConfirm(File from, File to)
    {
        try
        {
            createHardLink(from, to);
        }
        catch (FSWriteError ex)
        {
            throw ex;
        }
        catch (Throwable t)
        {
            throw new RuntimeException(String.format("Unable to hardlink from %s to %s", from, to), t);
        }
    }

    public static void createHardLinkWithoutConfirm(String from, String to)
    {
        createHardLinkWithoutConfirm(new File(from), new File(to));
    }

    public static void createHardLinkWithoutConfirm(File from, File to)
    {
        try
        {
            createHardLink(from, to);
        }
        catch (FSWriteError fse)
        {
            if (logger.isTraceEnabled())
                logger.trace("Could not hardlink file " + from + " to " + to, fse);
        }
    }

    public static void copyWithOutConfirm(String from, String to)
    {
        copyWithOutConfirm(new File(from), new File(to));
    }

    public static void copyWithOutConfirm(File from, File to)
    {
        try
        {
            Files.copy(from.toPath(), to.toPath());
        }
        catch (IOException e)
        {
            if (logger.isTraceEnabled())
                logger.trace("Could not copy file" + from + " to " + to, e);
        }
    }

    public static void copyWithConfirm(String from, String to)
    {
        copyWithConfirm(new File(from), new File(to));
    }

    public static void copyWithConfirm(File from, File to)
    {
        assert from.exists();
        if (logger.isTraceEnabled())
            logger.trace("Copying {} to {}", from.path(), to.path());

        try
        {
            Files.copy(from.toPath(), to.toPath());
        }
        catch (IOException e)
        {
            throw new FSWriteError(e, "Could not copy file" + from + " to " + to);
        }
    }

    public static void truncate(String path, long size)
    {
        File file = new File(path);
        try (FileChannel channel = file.newReadWriteChannel())
        {
            channel.truncate(size);
        }
        catch (IOException e)
        {
            throw PathUtils.propagateUnchecked(e, file.toPath(), true);
        }
    }

    public static void closeQuietly(Closeable c)
    {
        try
        {
            if (c != null)
                c.close();
        }
        catch (Exception e)
        {
            logger.warn("Failed closing {}", c, e);
        }
    }

    public static void closeQuietly(AutoCloseable c)
    {
        try
        {
            if (c != null)
                c.close();
        }
        catch (Exception e)
        {
            logger.warn("Failed closing {}", c, e);
        }
    }

    public static void close(Closeable... cs) throws IOException
    {
        close(Arrays.asList(cs));
    }

    public static void close(Iterable<? extends Closeable> cs) throws IOException
    {
        Throwable e = null;
        for (Closeable c : cs)
        {
            try
            {
                if (c != null)
                    c.close();
            }
            catch (Throwable ex)
            {
                if (e == null) e = ex;
                else e.addSuppressed(ex);
                logger.warn("Failed closing stream {}", c, ex);
            }
        }
        maybeFail(e, IOException.class);
    }

    public static void closeQuietly(Iterable<? extends AutoCloseable> cs)
    {
        for (AutoCloseable c : cs)
        {
            try
            {
                if (c != null)
                    c.close();
            }
            catch (Exception ex)
            {
                logger.warn("Failed closing {}", c, ex);
            }
        }
    }

    public static String getCanonicalPath(String filename)
    {
        return new File(filename).canonicalPath();
    }

    public static String getCanonicalPath(File file)
    {
        return file.canonicalPath();
    }

    /** Return true if file is contained in folder */
    public static boolean isContained(File folder, File file)
    {
        return folder.isAncestorOf(file);
    }

    public static void clean(ByteBuffer buffer)
    {
        if (buffer == null || !buffer.isDirect())
            return;

        // TODO Once we can get rid of Java 8, it's simpler to call sun.misc.Unsafe.invokeCleaner(ByteBuffer),
        // but need to take care of the attachment handling (i.e. whether 'buf' is a duplicate or slice) - that
        // is different in sun.misc.Unsafe.invokeCleaner and this implementation.

        try
        {
            Object cleaner = mhDirectBufferCleaner.bindTo(buffer).invoke();
            if (cleaner != null)
            {
                // ((DirectBuffer) buf).cleaner().clean();
                mhCleanerClean.bindTo(cleaner).invoke();
            }
        }
        catch (RuntimeException e)
        {
            throw e;
        }
        catch (Throwable e)
        {
            throw new RuntimeException(e);
        }
    }

    public static long parseFileSize(String value)
    {
        long result;
        if (!value.matches("\\d+(\\.\\d+)? (GiB|KiB|MiB|TiB|bytes)"))
        {
            throw new IllegalArgumentException(
                String.format("value %s is not a valid human-readable file size", value));
        }
        if (value.endsWith(" TiB"))
        {
            result = Math.round(Double.valueOf(value.replace(" TiB", "")) * ONE_TIB);
            return result;
        }
        else if (value.endsWith(" GiB"))
        {
            result = Math.round(Double.valueOf(value.replace(" GiB", "")) * ONE_GIB);
            return result;
        }
        else if (value.endsWith(" KiB"))
        {
            result = Math.round(Double.valueOf(value.replace(" KiB", "")) * ONE_KIB);
            return result;
        }
        else if (value.endsWith(" MiB"))
        {
            result = Math.round(Double.valueOf(value.replace(" MiB", "")) * ONE_MIB);
            return result;
        }
        else if (value.endsWith(" bytes"))
        {
            result = Math.round(Double.valueOf(value.replace(" bytes", "")));
            return result;
        }
        else
        {
            throw new IllegalStateException(String.format("FileUtils.parseFileSize() reached an illegal state parsing %s", value));
        }
    }

    public static String stringifyFileSize(double value)
    {
        double d;
        if (value >= ONE_TIB)
        {
            d = value / ONE_TIB;
            String val = df.format(d);
            return val + " TiB";
        }
        else if (value >= ONE_GIB)
        {
            d = value / ONE_GIB;
            String val = df.format(d);
            return val + " GiB";
        }
        else if (value >= ONE_MIB)
        {
            d = value / ONE_MIB;
            String val = df.format(d);
            return val + " MiB";
        }
        else if (value >= ONE_KIB)
        {
            d = value / ONE_KIB;
            String val = df.format(d);
            return val + " KiB";
        }
        else
        {
            String val = df.format(value);
            return val + " bytes";
        }
    }

    public static void handleCorruptSSTable(CorruptSSTableException e)
    {
        fsErrorHandler.get().ifPresent(handler -> handler.handleCorruptSSTable(e));
    }

    public static void handleFSError(FSError e)
    {
        fsErrorHandler.get().ifPresent(handler -> handler.handleFSError(e));
    }

    public static void handleStartupFSError(Throwable t)
    {
        fsErrorHandler.get().ifPresent(handler -> handler.handleStartupFSError(t));
    }

    /**
     * handleFSErrorAndPropagate will invoke the disk failure policy error handler,
     * which may or may not stop the daemon or transports. However, if we don't exit,
     * we still want to propagate the exception to the caller in case they have custom
     * exception handling
     *
     * @param e A filesystem error
     */
    public static void handleFSErrorAndPropagate(FSError e)
    {
        JVMStabilityInspector.inspectThrowable(e);
        throw e;
    }

    /**
     * Get the size of a directory in bytes
     * @param folder The directory for which we need size.
     * @return The size of the directory
     */
    public static long folderSize(File folder)
    {
        if (!folder.exists())
            return 0;

        final long [] sizeArr = {0L};
        try
        {
            Files.walkFileTree(folder.toPath(), new SimpleFileVisitor<Path>()
            {
                @Override
                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
                {
                    sizeArr[0] += attrs.size();
                    return FileVisitResult.CONTINUE;
                }

                @Override
                public FileVisitResult visitFileFailed(Path path, IOException e) throws IOException
                {
                    if (e instanceof NoSuchFileException)
                        return FileVisitResult.CONTINUE;
                    else
                        throw e;
                }
            });
        }
        catch (IOException e)
        {
            logger.error("Error while getting {} folder size. {}", folder, e.getMessage());
        }
        return sizeArr[0];
    }

    public static void append(File file, String ... lines)
    {
        if (file.exists())
            write(file, Arrays.asList(lines), StandardOpenOption.APPEND);
        else
            write(file, Arrays.asList(lines), StandardOpenOption.CREATE);
    }

    public static void appendAndSync(File file, String ... lines)
    {
        if (file.exists())
            write(file, Arrays.asList(lines), StandardOpenOption.APPEND, StandardOpenOption.SYNC);
        else
            write(file, Arrays.asList(lines), StandardOpenOption.CREATE, StandardOpenOption.SYNC);
    }

    public static void replace(File file, String ... lines)
    {
        write(file, Arrays.asList(lines), StandardOpenOption.TRUNCATE_EXISTING);
    }

    /**
     * Write lines to a file adding a newline to the end of each supplied line using the provided open options.
     *
     * If open option sync or dsync is provided this will not open the file with sync or dsync since it might end up syncing
     * many times for a lot of lines. Instead it will write all the lines and sync once at the end. Since the file is
     * never returned there is not much difference from the perspective of the caller.
     * @param file
     * @param lines
     * @param options
     */
    public static void write(File file, List<String> lines, StandardOpenOption ... options)
    {
        Set<StandardOpenOption> optionsSet = EnumSet.noneOf(StandardOpenOption.class);
        for (StandardOpenOption option : options)
            optionsSet.add(option);

        //Emulate the old FileSystemProvider.newOutputStream behavior for open options.
        if (optionsSet.isEmpty())
        {
            optionsSet.add(StandardOpenOption.CREATE);
            optionsSet.add(StandardOpenOption.TRUNCATE_EXISTING);
        }
        boolean sync = optionsSet.remove(StandardOpenOption.SYNC);
        boolean dsync = optionsSet.remove(StandardOpenOption.DSYNC);
        optionsSet.add(StandardOpenOption.WRITE);

        Path filePath = file.toPath();
        try (FileChannel fc = filePath.getFileSystem().provider().newFileChannel(filePath, optionsSet);
             BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(Channels.newOutputStream(fc), CHARSET.newEncoder())))
        {
            for (CharSequence line: lines) {
                writer.append(line);
                writer.newLine();
            }

            if (sync)
            {
                SyncUtil.force(fc, true);
            }
            else if (dsync)
            {
                SyncUtil.force(fc, false);
            }
        }
        catch (ClosedChannelException cce)
        {
            throw new RuntimeException(cce);
        }
        catch (IOException ex)
        {
            throw new FSWriteError(ex, file);
        }
    }

    public static List<String> readLines(File file)
    {
        try
        {
            return Files.readAllLines(file.toPath(), CHARSET);
        }
        catch (IOException ex)
        {
            if (ex instanceof NoSuchFileException)
                return Collections.emptyList();

            throw new RuntimeException(ex);
        }
    }

    public static void setFSErrorHandler(FSErrorHandler handler)
    {
        fsErrorHandler.getAndSet(Optional.ofNullable(handler));
    }

    @Deprecated
    public static void createDirectory(String directory)
    {
        createDirectory(new File(directory));
    }

    @Deprecated
    public static void createDirectory(File directory)
    {
        PathUtils.createDirectoriesIfNotExists(directory.toPath());
    }

    @Deprecated
    public static boolean delete(String file)
    {
        return new File(file).tryDelete();
    }

    @Deprecated
    public static void delete(File... files)
    {
        for (File file : files)
            file.tryDelete();
    }

    /**
     * Deletes all files and subdirectories under "dir".
     * @param dir Directory to be deleted
     * @throws FSWriteError if any part of the tree cannot be deleted
     */
    @Deprecated
    public static void deleteRecursiveWithThrottle(File dir, RateLimiter rateLimiter)
    {
        dir.deleteRecursive(rateLimiter);
    }

    /**
     * Deletes all files and subdirectories under "dir".
     * @param dir Directory to be deleted
     * @throws FSWriteError if any part of the tree cannot be deleted
     */
    @Deprecated
    public static void deleteRecursive(File dir)
    {
        dir.deleteRecursive();
    }

    /**
     * Schedules deletion of all file and subdirectories under "dir" on JVM shutdown.
     * @param dir Directory to be deleted
     */
    @Deprecated
    public static void deleteRecursiveOnExit(File dir)
    {
        dir.deleteRecursiveOnExit();
    }

    @Deprecated
    public static boolean isSubDirectory(File parent, File child)
    {
        return parent.isAncestorOf(child);
    }

    @Deprecated
    public static Throwable deleteWithConfirm(File file, Throwable accumulate)
    {
        return file.delete(accumulate, null);
    }

    @Deprecated
    public static Throwable deleteWithConfirm(File file, Throwable accumulate, RateLimiter rateLimiter)
    {
        return file.delete(accumulate, rateLimiter);
    }

    @Deprecated
    public static void deleteWithConfirm(String file)
    {
        deleteWithConfirm(new File(file));
    }

    @Deprecated
    public static void deleteWithConfirm(File file)
    {
        file.delete();
    }

    @Deprecated
    public static void renameWithOutConfirm(String from, String to)
    {
        new File(from).tryMove(new File(to));
    }

    @Deprecated
    public static void renameWithConfirm(String from, String to)
    {
        renameWithConfirm(new File(from), new File(to));
    }

    @Deprecated
    public static void renameWithConfirm(File from, File to)
    {
        from.move(to);
    }

    /**
     * Private constructor as the class contains only static methods.
     */
    private FileUtils()
    {
    }

    /**
     * Moves the contents of a directory to another directory.
     * <p>Once a file has been copied to the target directory it will be deleted from the source directory.
     * If a file already exists in the target directory a warning will be logged and the file will not
     * be deleted.</p>
     *
     * @param source the directory containing the files to move
     * @param target the directory where the files must be moved
     */
    public static void moveRecursively(Path source, Path target) throws IOException
    {
        logger.info("Moving {} to {}" , source, target);

        if (Files.isDirectory(source))
        {
            Files.createDirectories(target);

            for (File f : new File(source).tryList())
            {
                String fileName = f.name();
                moveRecursively(source.resolve(fileName), target.resolve(fileName));
            }

            deleteDirectoryIfEmpty(source);
        }
        else
        {
            if (Files.exists(target))
            {
                logger.warn("Cannot move the file {} to {} as the target file already exists." , source, target);
            }
            else
            {
                Files.copy(source, target, StandardCopyOption.COPY_ATTRIBUTES);
                Files.delete(source);
            }
        }
    }

    /**
     * Deletes the specified directory if it is empty
     *
     * @param path the path to the directory
     */
    public static void deleteDirectoryIfEmpty(Path path) throws IOException
    {
        Preconditions.checkArgument(Files.isDirectory(path), String.format("%s is not a directory", path));

        try
        {
            logger.info("Deleting directory {}", path);
            Files.delete(path);
        }
        catch (DirectoryNotEmptyException e)
        {
            try (Stream<Path> paths = Files.list(path))
            {
                String content = paths.map(p -> p.getFileName().toString()).collect(Collectors.joining(", "));

                logger.warn("Cannot delete the directory {} as it is not empty. (Content: {})", path, content);
            }
        }
    }
}