text: //===----- ModuleInterfaceBuilder.cpp - Compiles .swiftinterface files ----===//
  //
  // This source file is part of the Swift.org open source project
  //
  // Copyright (c) 2019 Apple Inc. and the Swift project authors
  // Licensed under Apache License v2.0 with Runtime Library Exception
  //
  // See https://swift.org/LICENSE.txt for license information
  // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
  //
  //===----------------------------------------------------------------------===//
  
  #define DEBUG_TYPE "textual-module-interface"
  
  #include "swift/Frontend/ModuleInterfaceLoader.h"
  #include "ModuleInterfaceBuilder.h"
  #include "swift/AST/ASTContext.h"
  #include "swift/AST/DiagnosticsFrontend.h"
  #include "swift/AST/DiagnosticsSema.h"
  #include "swift/AST/FileSystem.h"
  #include "swift/AST/Module.h"
  #include "llvm/ADT/StringSet.h"
  #include "swift/Basic/Defer.h"
  #include "swift/Frontend/Frontend.h"
  #include "swift/Frontend/ModuleInterfaceSupport.h"
  #include "swift/SILOptimizer/PassManager/Passes.h"
  #include "swift/Serialization/SerializationOptions.h"
  #include "clang/Frontend/CompilerInstance.h"
  #include "clang/Lex/PreprocessorOptions.h"
  #include "llvm/ADT/Hashing.h"
  #include "llvm/ADT/StringSet.h"
  #include "llvm/Support/xxhash.h"
  #include "llvm/Support/Debug.h"
  #include "llvm/Support/CommandLine.h"
  #include "llvm/Support/CrashRecoveryContext.h"
  #include "llvm/Support/Path.h"
  #include "llvm/Support/Errc.h"
  #include "llvm/Support/Regex.h"
  #include "llvm/Support/StringSaver.h"
  #include "llvm/Support/LockFileManager.h"
  
  using namespace swift;
  using FileDependency = SerializationOptions::FileDependency;
  namespace path = llvm::sys::path;
  
  /// If the file dependency in \p FullDepPath is inside the \p Base directory,
  /// this returns its path relative to \p Base. Otherwise it returns None.
  static Optional<StringRef> getRelativeDepPath(StringRef DepPath,
                                                StringRef Base) {
    // If Base is the root directory, or DepPath does not start with Base, bail.
    if (Base.size() <= 1 || !DepPath.startswith(Base)) {
      return None;
    }
  
    assert(DepPath.size() > Base.size() &&
        "should never depend on a directory");
  
    // Is the DepName something like ${Base}/foo.h"?
    if (path::is_separator(DepPath[Base.size()]))
      return DepPath.substr(Base.size() + 1);
  
    // Is the DepName something like "${Base}foo.h", where Base
    // itself contains a trailing slash?
    if (path::is_separator(Base.back()))
      return DepPath.substr(Base.size());
  
    // We have something next to Base, like "Base.h", that's somehow
    // become a dependency.
    return None;
  }
  
  bool ModuleInterfaceBuilder::collectDepsForSerialization(
      CompilerInstance &SubInstance, SmallVectorImpl<FileDependency> &Deps,
      bool IsHashBased) {
    llvm::vfs::FileSystem &fs = *sourceMgr.getFileSystem();
  
    auto &Opts = SubInstance.getASTContext().SearchPathOpts;
    SmallString<128> SDKPath(Opts.SDKPath);
    path::native(SDKPath);
    SmallString<128> ResourcePath(Opts.RuntimeResourcePath);
    path::native(ResourcePath);
  
    auto DTDeps = SubInstance.getDependencyTracker()->getDependencies();
    SmallVector<StringRef, 16> InitialDepNames(DTDeps.begin(), DTDeps.end());
    InitialDepNames.push_back(interfacePath);
    InitialDepNames.insert(InitialDepNames.end(),
                           extraDependencies.begin(), extraDependencies.end());
    llvm::StringSet<> AllDepNames;
    SmallString<128> Scratch;
  
    for (const auto &InitialDepName : InitialDepNames) {
      path::native(InitialDepName, Scratch);
      StringRef DepName = Scratch.str();
  
      assert(moduleCachePath.empty() || !DepName.startswith(moduleCachePath));
  
      // Serialize the paths of dependencies in the SDK relative to it.
      Optional<StringRef> SDKRelativePath = getRelativeDepPath(DepName, SDKPath);
      StringRef DepNameToStore = SDKRelativePath.getValueOr(DepName);
      bool IsSDKRelative = SDKRelativePath.hasValue();
  
      // Forwarding modules add the underlying prebuilt module to their
      // dependency list -- don't serialize that.
      if (!prebuiltCachePath.empty() && DepName.startswith(prebuiltCachePath))
        continue;
  
      if (AllDepNames.insert(DepName).second && dependencyTracker) {
        dependencyTracker->addDependency(DepName, /*isSystem*/IsSDKRelative);
      }
  
      // Don't serialize compiler-relative deps so the cache is relocatable.
      if (DepName.startswith(ResourcePath))
        continue;
  
      auto Status = fs.status(DepName);
      if (!Status)
        return true;
  
      /// Lazily load the dependency buffer if we need it. If we're not
      /// dealing with a hash-based dependencies, and if the dependency is
      /// not a .swiftmodule, we can avoid opening the buffer.
      std::unique_ptr<llvm::MemoryBuffer> DepBuf = nullptr;
      auto getDepBuf = [&]() -> llvm::MemoryBuffer * {
        if (DepBuf) return DepBuf.get();
        if (auto Buf = fs.getBufferForFile(DepName, /*FileSize=*/-1,
                                           /*RequiresNullTerminator=*/false)) {
          DepBuf = std::move(Buf.get());
          return DepBuf.get();
        }
        return nullptr;
      };
  
      if (IsHashBased) {
        auto buf = getDepBuf();
        if (!buf) return true;
        uint64_t hash = xxHash64(buf->getBuffer());
        Deps.push_back(
                       FileDependency::hashBased(DepNameToStore, IsSDKRelative,
                                                 Status->getSize(), hash));
      } else {
        uint64_t mtime =
        Status->getLastModificationTime().time_since_epoch().count();
        Deps.push_back(
                       FileDependency::modTimeBased(DepNameToStore, IsSDKRelative,
                                                    Status->getSize(), mtime));
      }
    }
    return false;
  }
  
  bool ModuleInterfaceBuilder::buildSwiftModuleInternal(
      StringRef OutPath, bool ShouldSerializeDeps,
      std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer) {
  
    auto outerPrettyStackState = llvm::SavePrettyStackState();
  
    bool SubError = false;
    bool RunSuccess = llvm::CrashRecoveryContext().RunSafelyOnThread([&] {
      // Pretend we're on the original thread for pretty-stack-trace purposes.
      auto savedInnerPrettyStackState = llvm::SavePrettyStackState();
      llvm::RestorePrettyStackState(outerPrettyStackState);
      SWIFT_DEFER {
        llvm::RestorePrettyStackState(savedInnerPrettyStackState);
      };
  
      SubError = subASTDelegate.runInSubCompilerInstance(moduleName,
                                                         interfacePath,
                                                         OutPath,
                                                         diagnosticLoc,
                                             [&](SubCompilerInstanceInfo &info) {
      auto &SubInstance = *info.Instance;
      auto subInvocation = SubInstance.getInvocation();
      FrontendOptions &FEOpts = subInvocation.getFrontendOptions();
      const auto &InputInfo = FEOpts.InputsAndOutputs.firstInput();
      StringRef InPath = InputInfo.file();
      const auto &OutputInfo =
      InputInfo.getPrimarySpecificPaths().SupplementaryOutputs;
      StringRef OutPath = OutputInfo.ModuleOutputPath;
  
      // Build the .swiftmodule; this is a _very_ abridged version of the logic
      // in performCompile in libFrontendTool, specialized, to just the one
      // module-serialization task we're trying to do here.
      LLVM_DEBUG(llvm::dbgs() << "Setting up instance to compile "
                 << InPath << " to " << OutPath << "\n");
  
      SWIFT_DEFER {
        // Make sure to emit a generic top-level error if a module fails to
        // load. This is not only good for users; it also makes sure that we've
        // emitted an error in the parent diagnostic engine, which is what
        // determines whether the process exits with a proper failure status.
        if (SubInstance.getASTContext().hadError()) {
          auto builtByCompiler =
              getSwiftInterfaceCompilerVersionForCurrentCompiler(
                  SubInstance.getASTContext());
          StringRef emittedByCompiler = info.CompilerVersion;
          diagnose(diag::module_interface_build_failed, moduleName,
                   emittedByCompiler == builtByCompiler, emittedByCompiler,
                   builtByCompiler);
        }
      };
  
      LLVM_DEBUG(llvm::dbgs() << "Performing sema\n");
      SubInstance.performSema();
      if (SubInstance.getASTContext().hadError()) {
        LLVM_DEBUG(llvm::dbgs() << "encountered errors\n");
        return true;
      }
  
      SILOptions &SILOpts = subInvocation.getSILOptions();
      auto Mod = SubInstance.getMainModule();
      auto &TC = SubInstance.getSILTypes();
      auto SILMod = performSILGeneration(Mod, TC, SILOpts);
      if (!SILMod) {
        LLVM_DEBUG(llvm::dbgs() << "SILGen did not produce a module\n");
        return true;
      }
  
      // Setup the callbacks for serialization, which can occur during the
      // optimization pipeline.
      SerializationOptions SerializationOpts;
      std::string OutPathStr = OutPath.str();
      SerializationOpts.OutputPath = OutPathStr.c_str();
      SerializationOpts.ModuleLinkName = FEOpts.ModuleLinkName;
      SerializationOpts.AutolinkForceLoad =
        !subInvocation.getIRGenOptions().ForceLoadSymbolName.empty();
  
      // Record any non-SDK module interface files for the debug info.
      StringRef SDKPath = SubInstance.getASTContext().SearchPathOpts.SDKPath;
      if (!getRelativeDepPath(InPath, SDKPath))
        SerializationOpts.ModuleInterface = InPath;
  
      SmallVector<FileDependency, 16> Deps;
      bool serializeHashes = FEOpts.SerializeModuleInterfaceDependencyHashes;
      if (collectDepsForSerialization(SubInstance, Deps, serializeHashes)) {
        return true;
      }
      if (ShouldSerializeDeps)
        SerializationOpts.Dependencies = Deps;
      SILMod->setSerializeSILAction([&]() {
        // We don't want to serialize module docs in the cache -- they
        // will be serialized beside the interface file.
        serializeToBuffers(Mod, SerializationOpts, ModuleBuffer,
                           /*ModuleDocBuffer*/nullptr,
                           /*SourceInfoBuffer*/nullptr,
                           SILMod.get());
      });
  
      LLVM_DEBUG(llvm::dbgs() << "Running SIL processing passes\n");
      if (SubInstance.performSILProcessing(SILMod.get())) {
        LLVM_DEBUG(llvm::dbgs() << "encountered errors\n");
        return true;
      }
      return SubInstance.getDiags().hadAnyError();
      });
    });
    return !RunSuccess || SubError;
  }
  
  bool ModuleInterfaceBuilder::buildSwiftModule(StringRef OutPath,
                                                bool ShouldSerializeDeps,
                            std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
                            llvm::function_ref<void()> RemarkRebuild) {
    auto build = [&]() {
      if (RemarkRebuild) {
        RemarkRebuild();
      }
      return buildSwiftModuleInternal(OutPath, ShouldSerializeDeps, ModuleBuffer);
    };
    if (disableInterfaceFileLock) {
      return build();
    }
    while (1) {
    // Attempt to lock the interface file. Only one process is allowed to build
    // module from the interface so we don't consume too much memory when multiple
    // processes are doing the same.
    // FIXME: We should surface the module building step to the build system so
    // we don't need to synchronize here.
    llvm::LockFileManager Locked(interfacePath);
    switch (Locked) {
    case llvm::LockFileManager::LFS_Error:{
      // ModuleInterfaceBuilder takes care of correctness and locks are only
      // necessary for performance. Fallback to building the module in case of any lock
      // related errors.
      if (RemarkRebuild) {
        diagnose(diag::interface_file_lock_failure, interfacePath);
      }
      // Clear out any potential leftover.
      Locked.unsafeRemoveLockFile();
      LLVM_FALLTHROUGH;
    }
    case llvm::LockFileManager::LFS_Owned: {
      return build();
    }
    case llvm::LockFileManager::LFS_Shared: {
      // Someone else is responsible for building the module. Wait for them to
      // finish.
      switch (Locked.waitForUnlock(256)) {
      case llvm::LockFileManager::Res_Success: {
        // This process may have a different module output path. If the other
        // process doesn't build the interface to this output path, we should try
        // building ourselves.
        auto bufferOrError = llvm::MemoryBuffer::getFile(OutPath);
        if (!bufferOrError)
          continue;
        if (ModuleBuffer)
          *ModuleBuffer = std::move(bufferOrError.get());
        return false;
      }
      case llvm::LockFileManager::Res_OwnerDied: {
        continue; // try again to get the lock.
      }
      case llvm::LockFileManager::Res_Timeout: {
        // Since ModuleInterfaceBuilder takes care of correctness, we try waiting for
        // another process to complete the build so swift does not do it done
        // twice. If case of timeout, build it ourselves.
        if (RemarkRebuild) {
          diagnose(diag::interface_file_lock_timed_out, interfacePath);
        }
        // Clear the lock file so that future invocations can make progress.
        Locked.unsafeRemoveLockFile();
        continue;
      }
      }
      break;
    }
    }
    }
  }
  