[Dependency Scanning] Add tracking of the number of dependency queries and emit them as remarks

This change adds collection of three metrics to the scanner:
- number of Swift module lookups
- number of named Clang module lookups
- recorded number of Clang modules which were imported into a Swift module by name

It re-uses the prior '-Rdependency-scan-cache', renaming it to '-Rdependency-scan' and adds emission of the above metrics as remarks when this flag is enabled. Followup changes will add further remarks about dependency scanner progress.

Author: artemcm

include/swift/AST/DiagnosticsSema.def

@@ -2432,6 +2432,22 @@ WARNING(dependency_scan_module_incompatible, none,
         "module file '%0' is incompatible with this Swift compiler: %1",
         (StringRef, StringRef))
 
+REMARK(dependency_scan_number_swift_queries, none,
+       "Number of Swift module queries: '%0'",
+       (uint32_t))
+REMARK(dependency_scan_number_named_clang_queries, none,
+       "Number of named Clang module queries: '%0'",
+       (uint32_t))
+REMARK(dependency_scan_number_swift_dependencies, none,
+       "Number of recorded Swift module dependencies: '%0'",
+       (uint32_t))
+REMARK(dependency_scan_number_named_clang_dependencies, none,
+       "Number of recorded Clang module dependencies queried by-name from a Swift client: '%0'",
+       (uint32_t))
+REMARK(dependency_scan_number_clang_dependencies, none,
+       "Number of recorded Clang module dependencies: '%0'",
+       (uint32_t))
+
 ERROR(clang_dependency_scan_error, none, "clang dependency scanning failure: %0", (StringRef))
 
 ERROR(clang_header_dependency_scan_error, none, "bridging header dependency scanning failure: %0", (StringRef))

include/swift/AST/ModuleDependencies.h

@@ -1111,6 +1111,11 @@ class ModuleDependenciesCache {
   bool hasDependency(StringRef moduleName) const;
   /// Whether we have cached dependency information for the given Swift module.
   bool hasSwiftDependency(StringRef moduleName) const;
+  /// Report the number of recorded Clang dependencies
+  int numberOfClangDependencies() const;
+  /// Report the number of recorded Swift dependencies
+  /// (Textual + Binary)
+  int numberOfSwiftDependencies() const;
 
   const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &
   getAlreadySeenClangModules() const {

include/swift/DependencyScan/ModuleDependencyScanner.h

@@ -37,6 +37,16 @@ using ImportStatementInfoMap =
     std::unordered_map<ModuleDependencyID,
                        std::vector<ScannerImportStatementInfo>>;
 
+struct ScannerMetrics {
+  /// Number of performed queries for a Swift dependency with a given name
+  std::atomic<uint32_t> SwiftModuleQueries;
+  /// Number of performed queries for a Clang dependency with a given name
+  std::atomic<uint32_t> NamedClangModuleQueries;
+  /// Number of discovered Clang module dependencies which are directly
+  /// imported from a Swift module by-name
+  std::atomic<uint32_t> RecordedNamedClangModuleDependencies;
+};
+
 /// A dependency scanning worker which performs filesystem lookup
 /// of a named module dependency.
 class ModuleDependencyScanningWorker {
@@ -48,6 +58,7 @@ class ModuleDependencyScanningWorker {
       DependencyTracker &DependencyTracker,
       std::shared_ptr<llvm::cas::ObjectStore> CAS,
       std::shared_ptr<llvm::cas::ActionCache> ActionCache,
+      std::shared_ptr<ScannerMetrics> ScanMetrics,
       llvm::PrefixMapper *mapper, DiagnosticEngine &diags);
 
 private:
@@ -128,6 +139,9 @@ class ModuleDependencyScanningWorker {
   std::shared_ptr<llvm::cas::ObjectStore> CAS;
   std::shared_ptr<llvm::cas::ActionCache> ActionCache;
 
+  // Metrics about actions performed by the scanner
+  std::shared_ptr<ScannerMetrics> scanMetrics;
+
   // Base command line invocation for clang scanner queries (both module and header)
   std::vector<std::string> clangScanningBaseCommandLineArgs;
   // Command line invocation for clang by-name module lookups
@@ -173,10 +187,11 @@ class SwiftDependencyTracker {
   std::map<std::string, FileEntry> TrackedFiles;
 };
 
-class ModuleDependencyIssueReporter {
+class DependencyScannerDiagnosticReporter {
 private:
-  ModuleDependencyIssueReporter(DiagnosticEngine &Diagnostics)
-      : Diagnostics(Diagnostics) {}
+  DependencyScannerDiagnosticReporter(DiagnosticEngine &Diagnostics,
+                                      bool EmitScanRemarks)
+      : Diagnostics(Diagnostics), EmitScanRemarks(EmitScanRemarks) {}
 
   /// Diagnose scanner failure and attempt to reconstruct the dependency
   /// path from the main module to the missing dependency
@@ -207,7 +222,12 @@ class ModuleDependencyIssueReporter {
       const std::vector<SwiftModuleScannerQueryResult::IncompatibleCandidate>
           &candidates);
 
+  /// Emit various metrics about the current scannig action
+  void emitScanMetrics(std::shared_ptr<ScannerMetrics> scanMetrics,
+                       const ModuleDependenciesCache &cache) const;
+
   DiagnosticEngine &Diagnostics;
+  bool EmitScanRemarks;
   std::unordered_set<std::string> ReportedMissing;
   // Restrict access to the parent scanner class.
   friend class ModuleDependencyScanner;
@@ -223,7 +243,8 @@ class ModuleDependencyScanner {
                           DependencyTracker &DependencyTracker,
                           std::shared_ptr<llvm::cas::ObjectStore> CAS,
                           std::shared_ptr<llvm::cas::ActionCache> ActionCache,
-                          DiagnosticEngine &Diagnostics, bool ParallelScan);
+                          DiagnosticEngine &Diagnostics, bool ParallelScan,
+                          bool EmitScanRemarks);
 
   /// Identify the scanner invocation's main module's dependencies
   llvm::ErrorOr<ModuleDependencyInfo>
@@ -347,7 +368,7 @@ class ModuleDependencyScanner {
   /// For the provided collection of unresolved imports
   /// belonging to identified Swift dependnecies, execute a parallel
   /// query to the Clang dependency scanner for each import's module identifier.
-  void performParallelClangModuleLookup(
+  void performClangModuleLookup(
       const ImportStatementInfoMap &unresolvedImportsMap,
       const ImportStatementInfoMap &unresolvedOptionalImportsMap,
       BatchClangModuleLookupResult &result);
@@ -396,7 +417,7 @@ class ModuleDependencyScanner {
 private:
   const CompilerInvocation &ScanCompilerInvocation;
   ASTContext &ScanASTContext;
-  ModuleDependencyIssueReporter IssueReporter;
+  DependencyScannerDiagnosticReporter ScanDiagnosticReporter;
 
   /// The location of where the explicitly-built modules will be output to
   std::string ModuleOutputPath;
@@ -406,6 +427,9 @@ class ModuleDependencyScanner {
   /// Reference to a module dependency cache
   ModuleDependenciesCache &DependencyCache;
 
+  /// When requested, collect scanning-related statistics
+  std::shared_ptr<ScannerMetrics> ScanMetrics;
+
   /// The available pool of workers for filesystem module search
   unsigned NumThreads;
   std::list<std::unique_ptr<ModuleDependencyScanningWorker>> Workers;

include/swift/Frontend/FrontendOptions.h

@@ -399,8 +399,8 @@ class FrontendOptions {
   /// The path at which to either serialize or deserialize the dependency scanner cache.
   std::string SerializedDependencyScannerCachePath;
 
-  /// Emit remarks indicating use of the serialized module dependency scanning cache.
-  bool EmitDependencyScannerCacheRemarks = false;
+  /// Emit dependency scanning related remarks.
+  bool EmitDependencyScannerRemarks = false;
 
   /// The path at which the dependency scanner can write generated files.
   std::string ScannerOutputDir;

include/swift/Option/FrontendOptions.td

@@ -292,8 +292,8 @@ def validate_prior_dependency_scan_cache : Flag<["-"], "validate-prior-dependenc
 def dependency_scan_cache_path : Separate<["-"], "dependency-scan-cache-path">,
   HelpText<"The path to output the dependency scanner's internal state.">;
 
-def dependency_scan_cache_remarks : Flag<["-"], "Rdependency-scan-cache">,
-  HelpText<"Emit remarks indicating use of the serialized module dependency scanning cache.">;
+def dependency_scan_remarks : Flag<["-"], "Rdependency-scan">,
+  HelpText<"Emit remarks for various steps taken by the dependency scanner.">;
 
 def parallel_scan : Flag<["-"], "parallel-scan">,
    HelpText<"Perform dependency scanning in-parallel.">;

lib/AST/ModuleDependencies.cpp

@@ -772,6 +772,14 @@ bool ModuleDependenciesCache::hasSwiftDependency(StringRef moduleName) const {
   return findSwiftDependency(moduleName).has_value();
 }
 
+int ModuleDependenciesCache::numberOfClangDependencies() const {
+  return ModuleDependenciesMap.at(ModuleDependencyKind::Clang).size();
+}
+int ModuleDependenciesCache::numberOfSwiftDependencies() const {
+  return ModuleDependenciesMap.at(ModuleDependencyKind::SwiftInterface).size() +
+         ModuleDependenciesMap.at(ModuleDependencyKind::SwiftBinary).size();
+}
+
 void ModuleDependenciesCache::recordDependency(
     StringRef moduleName, ModuleDependencyInfo dependency) {
   auto dependenciesKind = dependency.getKind();
@@ -908,7 +916,6 @@ void ModuleDependenciesCache::setSwiftOverlayDependencies(
     ModuleDependencyID moduleID,
     const ArrayRef<ModuleDependencyID> dependencyIDs) {
   auto dependencyInfo = findKnownDependency(moduleID);
-  assert(dependencyInfo.getSwiftOverlayDependencies().empty());
 #ifndef NDEBUG
   for (const auto &depID : dependencyIDs)
     assert(depID.Kind != ModuleDependencyKind::Clang);
@@ -923,7 +930,6 @@ void ModuleDependenciesCache::setCrossImportOverlayDependencies(
     ModuleDependencyID moduleID,
     const ModuleDependencyIDCollectionView dependencyIDs) {
   auto dependencyInfo = findKnownDependency(moduleID);
-  assert(dependencyInfo.getCrossImportOverlayDependencies().empty());
   // Copy the existing info to a mutable one we can then replace it with,
   // after setting its overlay dependencies.
   auto updatedDependencyInfo = dependencyInfo;