cassandra-trunk/src/java/org/apache/cassandra/io/sstable/Descriptor.java
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.cassandra.io.sstable;

import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Objects;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.sstable.metadata.IMetadataSerializer;
import org.apache.cassandra.io.sstable.metadata.MetadataSerializer;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.utils.Pair;

import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.cassandra.io.sstable.Component.separator;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;

/**
 * A SSTable is described by the keyspace and column family it contains data
 * for, an id (generation - where higher generations contain more recent data) and
 * an alphabetic version string.
 *
 * A descriptor can be marked as temporary, which influences generated filenames.
 */
public class Descriptor
{
    private static final Logger logger = LoggerFactory.getLogger(Descriptor.class);

    // Current SSTable directory format is {keyspace}/{tableName}-{tableId}[/backups|/snapshots/{tag}][/.{indexName}]/{component}.db
    // * {var} are mandatory components
    // * [var] are optional components
    static final Pattern SSTABLE_DIR_PATTERN = Pattern.compile(".*/(?<keyspace>\\w+)/" +
                                                               "(?<tableName>\\w+)-(?<tableId>[0-9a-f]{32})/" +
                                                               "(backups/|snapshots/(?<tag>[\\w-]+)/)?" +
                                                               "(\\.(?<indexName>[\\w-]+)/)?" +
                                                               "(?<component>[\\w-]+)\\.(?<ext>[\\w]+)$");

    // Pre 2.1 SSTable directory format is {keyspace}/{tableName}-{tableId}[/backups|/snapshots/{tag}][/.{indexName}]/{component}.db
    static final Pattern LEGACY_SSTABLE_DIR_PATTERN = Pattern.compile(".*/(?<keyspace>\\w+)/" +
                                                                      "(?<tableName>\\w+)/" +
                                                                      "(backups/|snapshots/(?<tag>[\\w-]+)/)?" +
                                                                      "(\\.(?<indexName>[\\w-]+)/)?" +
                                                                      "(?<component>[\\w-]+)\\.(?<ext>[\\w]+)$");

    private final static String LEGACY_TMP_REGEX_STR = "^((.*)\\-(.*)\\-)?tmp(link)?\\-((?:l|k).)\\-(\\d)*\\-(.*)$";
    private final static Pattern LEGACY_TMP_REGEX = Pattern.compile(LEGACY_TMP_REGEX_STR);

    public static String TMP_EXT = ".tmp";

    public static final char FILENAME_SEPARATOR = '-';

    private static final Splitter filenameSplitter = Splitter.on(FILENAME_SEPARATOR);

    /** canonicalized path to the directory where SSTable resides */
    public final File directory;
    /** version has the following format: <code>[a-z]+</code> */
    public final Version version;
    public final String ksname;
    public final String cfname;
    public final SSTableId id;
    private final int hashCode;
    private final String prefix;
    private final File baseFile;

    /**
     * A descriptor that assumes CURRENT_VERSION.
     */
    @VisibleForTesting
    public Descriptor(File directory, String ksname, String cfname, SSTableId id)
    {
        this(DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion(), directory, ksname, cfname, id);
    }

    /**
     * Constructor for sstable writers only.
     */
    public Descriptor(File directory, String ksname, String cfname, SSTableId id, SSTableFormat<?, ?> format)
    {
        this(format.getLatestVersion(), directory, ksname, cfname, id);
    }

    @VisibleForTesting
    public Descriptor(String version, File directory, String ksname, String cfname, SSTableId id, SSTableFormat<?, ?> format)
    {
        this(format.getVersion(version), directory, ksname, cfname, id);
    }

    public Descriptor(Version version, File directory, String ksname, String cfname, SSTableId id)
    {
        checkNotNull(version);
        checkNotNull(directory);
        checkNotNull(ksname);
        checkNotNull(cfname);

        this.version = version;
        this.directory = directory.toCanonical();
        this.ksname = ksname;
        this.cfname = cfname;
        this.id = id;

        StringBuilder buf = new StringBuilder();
        appendFileName(buf);
        this.prefix = buf.toString();
        this.baseFile = new File(directory.toPath().resolve(prefix));

        // directory is unnecessary for hashCode, and for simulator consistency we do not include it
        hashCode = Objects.hashCode(version, id, ksname, cfname);
    }

    private String tmpFilenameFor(Component component)
    {
        return fileFor(component) + TMP_EXT;
    }

    public File tmpFileFor(Component component)
    {
        return new File(directory.toPath().resolve(tmpFilenameFor(component)));
    }

    private String tmpFilenameForStreaming(Component component)
    {
        // Use UUID to handle concurrent streamings on the same sstable.
        // TMP_EXT allows temp file to be removed by {@link ColumnFamilyStore#scrubDataDirectories}
        return String.format("%s.%s%s", filenameFor(component), nextTimeUUID(), TMP_EXT);
    }

    /**
     * @return a unique temporary file name for given component during entire-sstable-streaming.
     */
    public File tmpFileForStreaming(Component component)
    {
        return new File(directory.toPath().resolve(tmpFilenameForStreaming(component)));
    }

    private String filenameFor(Component component)
    {
        return prefix + separator + component.name();
    }

    public File fileFor(Component component)
    {
        return new File(directory.toPath().resolve(filenameFor(component)));
    }

    public File baseFile()
    {
        return baseFile;
    }

    private void appendFileName(StringBuilder buff)
    {
        buff.append(version).append(separator);
        buff.append(id.toString());
        buff.append(separator).append(version.format.name());
    }

    public String relativeFilenameFor(Component component)
    {
        final StringBuilder buff = new StringBuilder();
        if (Directories.isSecondaryIndexFolder(directory))
        {
            buff.append(directory.name()).append(File.pathSeparator());
        }

        appendFileName(buff);
        buff.append(separator).append(component.name());
        return buff.toString();
    }

    public SSTableFormat<?, ?> getFormat()
    {
        return version.format;
    }

    /** Return any temporary files found in the directory */
    public List<File> getTemporaryFiles()
    {
        File[] tmpFiles = directory.tryList((dir, name) ->
                                              name.endsWith(Descriptor.TMP_EXT));

        List<File> ret = new ArrayList<>(tmpFiles.length);
        for (File tmpFile : tmpFiles)
            ret.add(tmpFile);

        return ret;
    }

    /**
     * Returns the set of components consisting of the provided mandatory components and those optional components
     * for which the corresponding file exists.
     */
    public Set<Component> getComponents(Set<Component> mandatory, Set<Component> optional)
    {
        ImmutableSet.Builder<Component> builder = ImmutableSet.builder();
        builder.addAll(mandatory);
        for (Component component : optional)
        {
            if (fileFor(component).exists())
                builder.add(component);
        }
        return builder.build();
    }

    public static boolean isValidFile(File file)
    {
        String filename = file.name();
        return filename.endsWith(".db") && !LEGACY_TMP_REGEX.matcher(filename).matches();
    }

    /**
     * Parse a sstable filename into a Descriptor.
     * <p>
     * SSTables files are all located within subdirectories of the form {@code <keyspace>/<table>/}. Normal sstables are
     * are directly within that subdirectory structure while 2ndary index, backups and snapshot are each inside an
     * additional subdirectory. The file themselves have the form:
     *   {@code <version>-<gen>-<format>-<component>}.
     * <p>
     * Note that this method will only sucessfully parse sstable files of supported versions.
     *
     * @param file the {@code File} object for the filename to parse.
     * @return the descriptor for the parsed file.
     *
     * @throws IllegalArgumentException if the provided {@code file} does point to a valid sstable filename. This could
     * mean either that the filename doesn't look like a sstable file, or that it is for an old and unsupported
     * versions.
     */
    public static Descriptor fromFile(File file)
    {
        return fromFileWithComponent(file).left;
    }

    public static Component componentFromFile(File file)
    {
        String name = file.name();
        List<String> tokens = filenameTokens(name);

        return Component.parse(tokens.get(3), formatFromName(name, tokens));
    }

    private static SSTableFormat<?, ?> formatFromName(String fileName, List<String> tokens)
    {
        String formatString = tokens.get(2);
        SSTableFormat<?, ?> format = DatabaseDescriptor.getSSTableFormats().get(formatString);
        if (format == null)
            throw invalidSSTable(fileName, "unknown 'format' part (%s)", formatString);
        return format;
    }

    /**
     * Parse a sstable filename, extracting both the {@code Descriptor} and {@code Component} part.
     * The keyspace/table name will be extracted from the directory path.
     *
     * @param file the {@code File} object for the filename to parse.
     * @return a pair of the descriptor and component corresponding to the provided {@code file}.
     *
     * @throws IllegalArgumentException if the provided {@code file} does point to a valid sstable filename. This could
     * mean either that the filename doesn't look like a sstable file, or that it is for an old and unsupported
     * versions.
     */
    public static Pair<Descriptor, Component> fromFileWithComponent(File file)
    {
        return fromFileWithComponent(file, true);
    }

    public static Pair<Descriptor, Component> fromFileWithComponent(File file, boolean validateDirs)
    {
        // We need to extract the keyspace and table names from the parent directories, so make sure we deal with the
        // absolute path.
        if (!file.isAbsolute())
            file = file.toAbsolute();

        SSTableInfo info = validateAndExtractInfo(file);
        String name = file.name();

        String keyspaceName = "";
        String tableName = "";

        Matcher sstableDirMatcher = SSTABLE_DIR_PATTERN.matcher(file.toString());

        // Use pre-2.1 SSTable format if current one does not match it
        if (!sstableDirMatcher.find(0))
        {
            sstableDirMatcher = LEGACY_SSTABLE_DIR_PATTERN.matcher(file.toString());
        }

        if (sstableDirMatcher.find(0))
        {
            keyspaceName = sstableDirMatcher.group("keyspace");
            tableName = sstableDirMatcher.group("tableName");
            String indexName = sstableDirMatcher.group("indexName");
            if (indexName != null)
            {
                tableName = String.format("%s.%s", tableName, indexName);
            }
        }
        else if (validateDirs)
        {
            logger.debug("Could not extract keyspace/table info from sstable directory {}", file.toString());
            throw invalidSSTable(name, String.format("cannot extract keyspace and table name from %s; make sure the sstable is in the proper sub-directories", file));
        }

        return Pair.create(new Descriptor(info.version, parentOf(name, file), keyspaceName, tableName, info.id), info.component);
    }

    /**
     * Parse a sstable filename, extracting both the {@code Descriptor} and {@code Component} part.
     *
     * @param file     the {@code File} object for the filename to parse.
     * @param keyspace The keyspace name of the file. If <code>null</code>, then the keyspace name will be extracted
     *                 from the directory path.
     * @param table    The table name of the file. If <code>null</code>, then the table name will be extracted from the
     *                 directory path.
     * @return a pair of the descriptor and component corresponding to the provided {@code file}.
     * @throws IllegalArgumentException if the provided {@code file} does point to a valid sstable filename. This could
     *                                  mean either that the filename doesn't look like a sstable file, or that it is for an old and unsupported
     *                                  versions.
     */
    public static Pair<Descriptor, Component> fromFileWithComponent(File file, String keyspace, String table)
    {
        if (null == keyspace || null == table)
        {
            return fromFileWithComponent(file);
        }

        SSTableInfo info = validateAndExtractInfo(file);
        return Pair.create(new Descriptor(info.version, parentOf(file.name(), file), keyspace, table, info.id), info.component);
    }

    private static List<String> filenameTokens(String name)
    {
        List<String> tokens = filenameSplitter.splitToList(name);
        int size = tokens.size();

        if (size != 4)
        {
            // This is an invalid sstable file for this version. But to provide a more helpful error message, we detect
            // old format sstable, which had the format:
            //   <keyspace>-<table>-(tmp-)?<version>-<gen>-<component>
            // Note that we assume it's an old format sstable if it has the right number of tokens: this is not perfect
            // but we're just trying to be helpful, not perfect.
            if (size == 5 || size == 6)
                throw new IllegalArgumentException(String.format("%s is of version %s which is now unsupported and cannot be read.", name, tokens.get(size - 3)));
            throw new IllegalArgumentException(String.format("Invalid sstable file %s: the name doesn't look like a supported sstable file name", name));
        }
        return tokens;
    }

    private static SSTableInfo validateAndExtractInfo(File file)
    {
        String name = file.name();
        List<String> tokens = filenameTokens(name);

        String versionString = tokens.get(0);
        if (!Version.validate(versionString))
            throw invalidSSTable(name, "invalid version %s", versionString);

        SSTableId id;
        try
        {
            id = SSTableIdFactory.instance.fromString(tokens.get(1));
        }
        catch (RuntimeException e)
        {
            throw invalidSSTable(name, "the 'id' part (%s) of the name doesn't parse as a valid unique identifier", tokens.get(1));
        }

        SSTableFormat<?, ?> format = formatFromName(name, tokens);
        Component component = Component.parse(tokens.get(3), format);

        Version version = format.getVersion(versionString);
        if (!version.isCompatible())
            throw invalidSSTable(name, "incompatible sstable version (%s); you should have run upgradesstables before upgrading", versionString);

        return new SSTableInfo(version, id, component);
    }

    private static class SSTableInfo
    {
        final Version version;
        final SSTableId id;
        final Component component;

        SSTableInfo(Version version, SSTableId id, Component component)
        {
            this.version = version;
            this.id = id;
            this.component = component;
        }
    }

    private static File parentOf(String name, File file)
    {
        File parent = file.parent();
        if (parent == null)
            throw invalidSSTable(name, "cannot extract keyspace and table name; make sure the sstable is in the proper sub-directories");
        return parent;
    }

    private static IllegalArgumentException invalidSSTable(String name, String msgFormat, Object... parameters)
    {
        throw new IllegalArgumentException(String.format("Invalid sstable file " + name + ": " + msgFormat, parameters));
    }

    public IMetadataSerializer getMetadataSerializer()
    {
        return new MetadataSerializer();
    }

    /**
     * @return true if the current Cassandra version can read the given sstable version
     */
    public boolean isCompatible()
    {
        return version.isCompatible();
    }

    public Set<Component> discoverComponents()
    {
        Set<Component> components = Sets.newHashSetWithExpectedSize(Component.Type.all.size());
        for (Component component : Component.getSingletonsFor(version.format))
        {
            if (fileFor(component).exists())
                components.add(component);
        }
        return components;
    }

    @Override
    public String toString()
    {
        return baseFile().absolutePath();
    }

    @Override
    public boolean equals(Object o)
    {
        if (o == this)
            return true;
        if (!(o instanceof Descriptor))
            return false;
        Descriptor that = (Descriptor)o;
        if (this.hashCode != that.hashCode)
            return false;
        return that.directory.equals(this.directory)
                       && that.id.equals(this.id)
                       && that.ksname.equals(this.ksname)
                       && that.cfname.equals(this.cfname)
                       && that.version.equals(this.version);
    }

    @Override
    public int hashCode()
    {
        return hashCode;
    }
}
