[Dependency Scanning] Re-use cached queried Clang dependency info in subsequent scanning stages

Dependency Scanning is recursive over discovered Swift module overlays and cross-import overlays. In the main 'resolveImportedModuleDependencies' routine, after all Swift and Clang dependencies of the main module are discovered, the scanner looks up Swift overlays for discovered Clang modules. For each such Swift overlay, the scanner will then proceed to call 'resolveImportedModuleDependencies' on the overlay module.

On these subsequent recursive calls to 'resolveImportedModuleDependencies', Clang dependency resolution will re-query all imports of the overlay module which do not resolve to Swift modules, even if they had been queried previously in a parent invocation. This change adds the ability to re-use previously-queried-and-cached Clang module dependency information by having the dependency cache also store the set of visible modules which resulted from each by-name lookup.

Author: artemcm

include/swift/AST/ModuleDependencies.h

@@ -280,7 +280,7 @@ class ModuleDependencyInfoStorageBase {
   /// A list of Clang modules that are visible to this Swift module. This
   /// includes both direct Clang modules as well as transitive Clang
   /// module dependencies when they are exported
-  llvm::StringSet<> visibleClangModules;
+  std::vector<std::string> bridgingHeaderVisibleClangModules;
 
   /// ModuleDependencyInfo is finalized (with all transitive dependencies
   /// and inputs).
@@ -893,14 +893,12 @@ class ModuleDependencyInfo {
       llvm_unreachable("Unexpected module dependency kind");
   }
 
-  llvm::StringSet<> &getVisibleClangModules() const {
-    return storage->visibleClangModules;
+  ArrayRef<std::string> getHeaderVisibleClangModules() const {
+    return storage->bridgingHeaderVisibleClangModules;
   }
-
-  void
-  addVisibleClangModules(const std::vector<std::string> &moduleNames) const {
-    storage->visibleClangModules.insert(moduleNames.begin(),
-                                        moduleNames.end());
+  void setHeaderVisibleClangModules(
+      const std::vector<std::string> &moduleNames) const {
+    storage->bridgingHeaderVisibleClangModules = moduleNames;
   }
 
   /// Whether explicit input paths of all the module dependencies
@@ -1073,6 +1071,9 @@ class ModuleDependenciesCache {
 private:
   /// Discovered dependencies
   ModuleDependenciesKindMap ModuleDependenciesMap;
+  /// A map from Clang module name to all visible modules to a client
+  /// of a by-name import of this Clang module
+  llvm::StringMap<std::vector<std::string>> clangModulesVisibleFromNamedLookup;
   /// Set containing all of the Clang modules that have already been seen.
   llvm::DenseSet<clang::tooling::dependencies::ModuleID> alreadySeenClangModules;
   /// Name of the module under scan
@@ -1111,6 +1112,8 @@ class ModuleDependenciesCache {
   bool hasDependency(StringRef moduleName) const;
   /// Whether we have cached dependency information for the given Swift module.
   bool hasSwiftDependency(StringRef moduleName) const;
+  /// Whether we have cached dependency information for the given Clang module.
+  bool hasClangDependency(StringRef moduleName) const;
   /// Report the number of recorded Clang dependencies
   int numberOfClangDependencies() const;
   /// Report the number of recorded Swift dependencies
@@ -1152,9 +1155,20 @@ class ModuleDependenciesCache {
   /// Query all cross-import overlay dependencies
   llvm::ArrayRef<ModuleDependencyID>
   getCrossImportOverlayDependencies(const ModuleDependencyID &moduleID) const;
+
   /// Query all visible Clang modules for a given Swift dependency
-  llvm::StringSet<>&
-  getVisibleClangModules(ModuleDependencyID moduleID) const;
+  llvm::StringSet<>
+  getAllVisibleClangModules(ModuleDependencyID moduleID) const;
+  /// Query all Clang modules visible via a by-name lookup of given
+  /// Clang dependency
+  llvm::ArrayRef<std::string>
+  getVisibleClangModulesFromLookup(StringRef moduleName) const;
+  bool hasVisibleClangModulesFromLookup(StringRef moduleName) const;
+  /// Query all Clang modules visible from a given Swift module's
+  /// bridging header
+  llvm::ArrayRef<std::string>
+  getVisibleClangModulesViaHeader(ModuleDependencyID moduleID) const;
+  bool hasVisibleClangModulesViaHeader(ModuleDependencyID moduleID) const;
 
   /// Look for module dependencies for a module with the given ID
   ///
@@ -1230,8 +1244,8 @@ class ModuleDependenciesCache {
     const ModuleDependencyIDCollectionView dependencyIDs);
   /// Add to this module's set of visible Clang modules
   void
-  addVisibleClangModules(ModuleDependencyID moduleID,
-                         const std::vector<std::string> &moduleNames);
+  setVisibleClangModulesFromLookup(ModuleDependencyID clangModuleID,
+                                   const std::vector<std::string> &moduleNames);
 
   StringRef getMainModuleName() const { return mainScanModuleName; }
 

include/swift/DependencyScan/ModuleDependencyScanner.h

@@ -37,6 +37,14 @@ using ImportStatementInfoMap =
     std::unordered_map<ModuleDependencyID,
                        std::vector<ScannerImportStatementInfo>>;
 
+/// A map from a module ID to a collection of module IDs.
+using ModuleIDToModuleIDSetVectorMap =
+    std::unordered_map<ModuleDependencyID,
+                       ModuleDependencyIDSetVector>;
+
+using ModuleIDImportInfoPair =
+    std::pair<ModuleDependencyID, ScannerImportStatementInfo>;
+
 struct ScannerMetrics {
   /// Number of performed queries for a Swift dependency with a given name
   std::atomic<uint32_t> SwiftModuleQueries;
@@ -306,7 +314,7 @@ class ModuleDependencyScanner {
   void resolveSwiftModuleDependencies(
       const ModuleDependencyID &rootModuleID,
       ModuleDependencyIDSetVector &discoveredSwiftModules);
-  void resolveAllClangModuleDependencies(
+  void resolveClangModuleDependencies(
       ArrayRef<ModuleDependencyID> swiftModules,
       ModuleDependencyIDSetVector &discoveredClangModules);
   void resolveHeaderDependencies(
@@ -391,25 +399,19 @@ class ModuleDependencyScanner {
   ///    in \c failedToResolveImports.
   /// 4. Update the set of resolved Clang dependencies for each Swift
   ///    module dependency in \c resolvedClangDependenciesMap.
-  void cacheComputedClangModuleLookupResults(
+  void processBatchClangModuleQueryResult(
       const BatchClangModuleLookupResult &lookupResult,
       const ImportStatementInfoMap &unresolvedImportsMap,
       const ImportStatementInfoMap &unresolvedOptionalImportsMap,
-      ArrayRef<ModuleDependencyID> swiftModuleDependents,
       ModuleDependencyIDSetVector &allDiscoveredClangModules,
-      std::vector<std::pair<ModuleDependencyID, ScannerImportStatementInfo>>
-          &failedToResolveImports,
-      std::unordered_map<ModuleDependencyID, ModuleDependencyIDSetVector>
-          &resolvedClangDependenciesMap);
+      std::vector<ModuleIDImportInfoPair> &failedToResolveImports,
+      ModuleIDToModuleIDSetVectorMap &resolvedClangDependenciesMap);
 
   /// Re-query some failed-to-resolve Clang imports from cache
   /// in chance they were brought in as transitive dependencies.
   void reQueryMissedModulesFromCache(
-      const std::vector<
-          std::pair<ModuleDependencyID, ScannerImportStatementInfo>>
-          &failedToResolveImports,
-      std::unordered_map<ModuleDependencyID, ModuleDependencyIDSetVector>
-          &resolvedClangDependenciesMap);
+      const std::vector<ModuleIDImportInfoPair> &failedToResolveImports,
+      ModuleIDToModuleIDSetVectorMap &resolvedClangDependenciesMap);
 
   /// Assuming the \c `moduleImport` failed to resolve,
   /// iterate over all binary Swift module dependencies with serialized

lib/AST/ModuleDependencies.cpp

@@ -771,6 +771,9 @@ bool ModuleDependenciesCache::hasDependency(StringRef moduleName) const {
 bool ModuleDependenciesCache::hasSwiftDependency(StringRef moduleName) const {
   return findSwiftDependency(moduleName).has_value();
 }
+bool ModuleDependenciesCache::hasClangDependency(StringRef moduleName) const {
+  return findDependency(moduleName, ModuleDependencyKind::Clang).has_value();
+}
 
 int ModuleDependenciesCache::numberOfClangDependencies() const {
   return ModuleDependenciesMap.at(ModuleDependencyKind::Clang).size();
@@ -856,6 +859,16 @@ void ModuleDependenciesCache::recordClangDependency(
   }
 }
 
+void ModuleDependenciesCache::setVisibleClangModulesFromLookup(
+    ModuleDependencyID moduleID,
+    const std::vector<std::string> &visibleModules) {
+  ASSERT(moduleID.Kind == ModuleDependencyKind::Clang);
+  if (visibleModules.empty())
+    return;
+
+  clangModulesVisibleFromNamedLookup[moduleID.ModuleName] = visibleModules;
+}
+
 void ModuleDependenciesCache::updateDependency(
     ModuleDependencyID moduleID, ModuleDependencyInfo dependencyInfo) {
   auto &map = getDependenciesMap(moduleID.Kind);
@@ -948,22 +961,55 @@ ModuleDependencyIDCollectionView ModuleDependenciesCache::getAllDependencies(
       moduleInfo.getImportedClangDependencies());
 }
 
-void ModuleDependenciesCache::addVisibleClangModules(
-    ModuleDependencyID moduleID, const std::vector<std::string> &moduleNames) {
-  if (moduleNames.empty())
-    return;
-  auto dependencyInfo = findKnownDependency(moduleID);
-  auto updatedDependencyInfo = dependencyInfo;
-  updatedDependencyInfo.addVisibleClangModules(moduleNames);
-  updateDependency(moduleID, updatedDependencyInfo);
+llvm::StringSet<> ModuleDependenciesCache::getAllVisibleClangModules(
+    ModuleDependencyID moduleID) const {
+  ASSERT(moduleID.Kind == ModuleDependencyKind::SwiftSource ||
+         moduleID.Kind == ModuleDependencyKind::SwiftInterface ||
+         moduleID.Kind == ModuleDependencyKind::SwiftBinary);
+  llvm::StringSet<> result;
+  if (hasVisibleClangModulesViaHeader(moduleID)) {
+    auto headerVisibleModules = getVisibleClangModulesViaHeader(moduleID);
+    result.insert(headerVisibleModules.begin(), headerVisibleModules.end());
+  }
+  for (const auto &clangDepID :
+       findKnownDependency(moduleID).getImportedClangDependencies()) {
+    // FIXME: This scenario should not occur
+    // assert(hasVisibleClangModulesFrom(clangDepID.ModuleName));
+    if (hasVisibleClangModulesFromLookup(clangDepID.ModuleName)) {
+      auto visibleModulesViaImport =
+          getVisibleClangModulesFromLookup(clangDepID.ModuleName);
+      result.insert(visibleModulesViaImport.begin(),
+                    visibleModulesViaImport.end());
+    }
+  }
+  return result;
 }
 
-llvm::StringSet<> &ModuleDependenciesCache::getVisibleClangModules(
+ArrayRef<std::string> ModuleDependenciesCache::getVisibleClangModulesFromLookup(
+    StringRef moduleName) const {
+  ASSERT(hasVisibleClangModulesFromLookup(moduleName));
+  return clangModulesVisibleFromNamedLookup.at(moduleName);
+}
+
+bool ModuleDependenciesCache::hasVisibleClangModulesFromLookup(
+    StringRef moduleName) const {
+  return clangModulesVisibleFromNamedLookup.contains(moduleName);
+}
+
+llvm::ArrayRef<std::string>
+ModuleDependenciesCache::getVisibleClangModulesViaHeader(
+    ModuleDependencyID moduleID) const {
+  ASSERT(moduleID.Kind == ModuleDependencyKind::SwiftSource ||
+         moduleID.Kind == ModuleDependencyKind::SwiftInterface ||
+         moduleID.Kind == ModuleDependencyKind::SwiftBinary);
+  return findKnownDependency(moduleID).getHeaderVisibleClangModules();
+}
+bool ModuleDependenciesCache::hasVisibleClangModulesViaHeader(
     ModuleDependencyID moduleID) const {
   ASSERT(moduleID.Kind == ModuleDependencyKind::SwiftSource ||
          moduleID.Kind == ModuleDependencyKind::SwiftInterface ||
          moduleID.Kind == ModuleDependencyKind::SwiftBinary);
-  return findKnownDependency(moduleID).getVisibleClangModules();
+  return !getVisibleClangModulesViaHeader(moduleID).empty();
 }
 
 ModuleDependencyIDCollectionView