Optimize representation of runfiles in compact execution log

src/main/java/com/google/devtools/build/lib/actions/ActionExecutionContext.java

@@ -20,6 +20,7 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.devtools.build.lib.actions.cache.OutputMetadataStore;
+import com.google.devtools.build.lib.analysis.SymlinkEntry;
 import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue.RunfileSymlinksMode;
 import com.google.devtools.build.lib.bugreport.BugReporter;
 import com.google.devtools.build.lib.clock.Clock;
@@ -88,6 +89,37 @@ public boolean isBuildRunfileLinks() {
     public String getWorkspaceName() {
       return wrapped.getWorkspaceName();
     }
+
+    @Override
+    public NestedSet<Artifact> getArtifactsAtCanonicalLocationsForLogging() {
+      return wrapped.getArtifactsAtCanonicalLocationsForLogging();
+    }
+
+    @Override
+    public Iterable<PathFragment> getEmptyFilenamesForLogging() {
+      return wrapped.getEmptyFilenamesForLogging();
+    }
+
+    @Override
+    public NestedSet<SymlinkEntry> getSymlinksForLogging() {
+      return wrapped.getSymlinksForLogging();
+    }
+
+    @Override
+    public NestedSet<SymlinkEntry> getRootSymlinksForLogging() {
+      return wrapped.getRootSymlinksForLogging();
+    }
+
+    @Nullable
+    @Override
+    public Artifact getRepoMappingManifestForLogging() {
+      return wrapped.getRepoMappingManifestForLogging();
+    }
+
+    @Override
+    public boolean isLegacyExternalRunfiles() {
+      return wrapped.isLegacyExternalRunfiles();
+    }
   }
 
   /**

src/main/java/com/google/devtools/build/lib/actions/BUILD

@@ -163,6 +163,7 @@ java_library(
         ":thread_state_receiver",
         "//src/main/java/com/google/devtools/build/lib/analysis:config/build_configuration",
         "//src/main/java/com/google/devtools/build/lib/analysis:config/core_options",
+        "//src/main/java/com/google/devtools/build/lib/analysis:symlink_entry",
         "//src/main/java/com/google/devtools/build/lib/analysis/platform",
         "//src/main/java/com/google/devtools/build/lib/bugreport",
         "//src/main/java/com/google/devtools/build/lib/buildeventstream",
@@ -553,8 +554,10 @@ java_library(
     deps = [
         ":artifacts",
         "//src/main/java/com/google/devtools/build/lib/analysis:config/build_configuration",
+        "//src/main/java/com/google/devtools/build/lib/analysis:symlink_entry",
         "//src/main/java/com/google/devtools/build/lib/collect/nestedset",
         "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
+        "//third_party:jsr305",
     ],
 )
 

src/main/java/com/google/devtools/build/lib/actions/RunfilesTree.java

@@ -14,10 +14,12 @@
 
 package com.google.devtools.build.lib.actions;
 
+import com.google.devtools.build.lib.analysis.SymlinkEntry;
 import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue.RunfileSymlinksMode;
 import com.google.devtools.build.lib.collect.nestedset.NestedSet;
 import com.google.devtools.build.lib.vfs.PathFragment;
 import java.util.Map;
+import javax.annotation.Nullable;
 
 /** Lazy wrapper for a single runfiles tree. */
 // TODO(bazel-team): Ideally we could refer to Runfiles objects directly here, but current package
@@ -45,4 +47,32 @@ public interface RunfilesTree {
 
   /** Returns the name of the workspace that the build is occurring in. */
   String getWorkspaceName();
+
+  /**
+   * Returns artifacts the runfiles tree contain symlinks to at their canonical locations.
+   *
+   * <p>This does <b>not</b> include artifacts that only the symlinks and root symlinks point to.
+   */
+  NestedSet<Artifact> getArtifactsAtCanonicalLocationsForLogging();
+
+  /**
+   * Returns the set of names of implicit empty files to materialize.
+   *
+   * <p>If this runfiles tree does not implicitly add empty files, implementations should have a
+   * dedicated fast path that returns an empty set without traversing the tree.
+   */
+  Iterable<PathFragment> getEmptyFilenamesForLogging();
+
+  /** Returns the set of custom symlink entries. */
+  NestedSet<SymlinkEntry> getSymlinksForLogging();
+
+  /** Returns the set of root symlinks. */
+  NestedSet<SymlinkEntry> getRootSymlinksForLogging();
+
+  /** Returns the repo mapping manifest if it exists. */
+  @Nullable
+  Artifact getRepoMappingManifestForLogging();
+
+  /** Whether this runfiles tree materializes external runfiles also at their legacy locations. */
+  boolean isLegacyExternalRunfiles();
 }

src/main/java/com/google/devtools/build/lib/analysis/Runfiles.java

@@ -245,10 +245,17 @@ public NestedSet<SymlinkEntry> getSymlinks() {
 
   @Override
   public Depset /*<String>*/ getEmptyFilenamesForStarlark() {
-    return Depset.of(String.class, getEmptyFilenames());
+    return Depset.of(
+        String.class,
+        NestedSetBuilder.wrap(
+            Order.STABLE_ORDER,
+            Iterables.transform(getEmptyFilenames(), PathFragment::getPathString)));
   }
 
-  public NestedSet<String> getEmptyFilenames() {
+  public Iterable<PathFragment> getEmptyFilenames() {
+    if (emptyFilesSupplier == DUMMY_EMPTY_FILES_SUPPLIER) {
+      return ImmutableList.of();
+    }
     Set<PathFragment> manifestKeys =
         Streams.concat(
                 symlinks.toList().stream().map(SymlinkEntry::getPath),
@@ -259,13 +266,7 @@ public NestedSet<String> getEmptyFilenames() {
                                 ? artifact.getOutputDirRelativePath(false)
                                 : artifact.getRunfilesPath()))
             .collect(ImmutableSet.toImmutableSet());
-    Iterable<PathFragment> emptyKeys = emptyFilesSupplier.getExtraPaths(manifestKeys);
-    return NestedSetBuilder.<String>stableOrder()
-        .addAll(
-            Streams.stream(emptyKeys)
-                .map(PathFragment::toString)
-                .collect(ImmutableList.toImmutableList()))
-        .build();
+    return emptyFilesSupplier.getExtraPaths(manifestKeys);
   }
 
   /**
@@ -383,6 +384,10 @@ public Map<PathFragment, Artifact> getRunfilesInputs(
     return builder.build();
   }
 
+  public boolean isLegacyExternalRunfiles() {
+    return legacyExternalRunfiles;
+  }
+
   /**
    * Helper class to handle munging the paths of external artifacts.
    */

src/main/java/com/google/devtools/build/lib/analysis/RunfilesSupport.java

@@ -88,14 +88,15 @@ public final class RunfilesSupport {
   private static final String OUTPUT_MANIFEST_BASENAME = "MANIFEST";
   private static final String REPO_MAPPING_MANIFEST_EXT = ".repo_mapping";
 
-  private static class RunfilesTreeImpl implements RunfilesTree {
+  @VisibleForTesting
+  public static class RunfilesTreeImpl implements RunfilesTree {
 
     private static final WeakReference<Map<PathFragment, Artifact>> NOT_YET_COMPUTED =
         new WeakReference<>(null);
 
     private final PathFragment execPath;
     private final Runfiles runfiles;
-    private final Artifact repoMappingManifest;
+    @Nullable private final Artifact repoMappingManifest;
 
     /**
      * The cached runfiles mapping. Possible values:
@@ -119,7 +120,7 @@ private static class RunfilesTreeImpl implements RunfilesTree {
     private RunfilesTreeImpl(
         PathFragment execPath,
         Runfiles runfiles,
-        Artifact repoMappingManifest,
+        @Nullable Artifact repoMappingManifest,
         boolean buildRunfileLinks,
         boolean cacheMapping,
         RunfileSymlinksMode runfileSymlinksMode) {
@@ -131,6 +132,17 @@ private RunfilesTreeImpl(
       this.cachedMapping = cacheMapping ? NOT_YET_COMPUTED : null;
     }
 
+    @VisibleForTesting
+    public RunfilesTreeImpl(PathFragment execPath, Runfiles runfiles) {
+      this(
+          execPath,
+          runfiles,
+          /* repoMappingManifest= */ null,
+          /* buildRunfileLinks= */ false,
+          /* cacheMapping= */ false,
+          RunfileSymlinksMode.EXTERNAL);
+    }
+
     @Override
     public PathFragment getExecPath() {
       return execPath;
@@ -167,6 +179,37 @@ public NestedSet<Artifact> getArtifacts() {
       return runfiles.getAllArtifacts();
     }
 
+    @Override
+    public NestedSet<Artifact> getArtifactsAtCanonicalLocationsForLogging() {
+      return runfiles.getArtifacts();
+    }
+
+    @Override
+    public Iterable<PathFragment> getEmptyFilenamesForLogging() {
+      return runfiles.getEmptyFilenames();
+    }
+
+    @Override
+    public NestedSet<SymlinkEntry> getSymlinksForLogging() {
+      return runfiles.getSymlinks();
+    }
+
+    @Override
+    public NestedSet<SymlinkEntry> getRootSymlinksForLogging() {
+      return runfiles.getRootSymlinks();
+    }
+
+    @Nullable
+    @Override
+    public Artifact getRepoMappingManifestForLogging() {
+      return repoMappingManifest;
+    }
+
+    @Override
+    public boolean isLegacyExternalRunfiles() {
+      return runfiles.isLegacyExternalRunfiles();
+    }
+
     @Override
     public RunfileSymlinksMode getSymlinksMode() {
       return runfileSymlinksMode;

src/main/java/com/google/devtools/build/lib/bazel/BUILD

@@ -136,6 +136,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/exec:executor_builder",
         "//src/main/java/com/google/devtools/build/lib/exec:module_action_context_registry",
         "//src/main/java/com/google/devtools/build/lib/exec:spawn_log_context",
+        "//src/main/java/com/google/devtools/build/lib/packages/semantics",
         "//src/main/java/com/google/devtools/build/lib/remote/options",
         "//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception",
         "//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code",

src/main/java/com/google/devtools/build/lib/bazel/SpawnLogModule.java

@@ -27,6 +27,7 @@
 import com.google.devtools.build.lib.exec.ExpandedSpawnLogContext.Encoding;
 import com.google.devtools.build.lib.exec.ModuleActionContextRegistry;
 import com.google.devtools.build.lib.exec.SpawnLogContext;
+import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions;
 import com.google.devtools.build.lib.remote.options.RemoteOptions;
 import com.google.devtools.build.lib.runtime.BlazeModule;
 import com.google.devtools.build.lib.runtime.CommandEnvironment;
@@ -104,6 +105,10 @@ private void initOutputs(CommandEnvironment env) throws IOException {
             new CompactSpawnLogContext(
                 outputPath,
                 env.getExecRoot().asFragment(),
+                env.getWorkspaceName(),
+                env.getOptions()
+                    .getOptions(BuildLanguageOptions.class)
+                    .experimentalSiblingRepositoryLayout,
                 env.getOptions().getOptions(RemoteOptions.class),
                 env.getRuntime().getFileSystem().getDigestFunction(),
                 env.getXattrProvider());

src/main/java/com/google/devtools/build/lib/exec/BUILD

@@ -277,6 +277,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/actions:artifacts",
         "//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
         "//src/main/java/com/google/devtools/build/lib/actions:runfiles_tree",
+        "//src/main/java/com/google/devtools/build/lib/analysis:symlink_entry",
         "//src/main/java/com/google/devtools/build/lib/analysis/platform:platform_utils",
         "//src/main/java/com/google/devtools/build/lib/cmdline",
         "//src/main/java/com/google/devtools/build/lib/collect/nestedset",
@@ -305,9 +306,11 @@ java_library(
         "StableSort.java",
     ],
     deps = [
+        "//src/main/java/com/google/devtools/build/lib/cmdline",
         "//src/main/java/com/google/devtools/build/lib/profiler",
         "//src/main/java/com/google/devtools/build/lib/util:pair",
         "//src/main/java/com/google/devtools/build/lib/util/io:io-proto",
+        "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
         "//src/main/protobuf:spawn_java_proto",
         "//third_party:guava",
         "//third_party:jsr305",