Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimize representation of runfiles in compact execution log #23321

Closed
wants to merge 51 commits into from
Closed
Changes from 1 commit
Commits
Show all changes
51 commits
Select commit Hold shift + click to select a range
7b47753
Add RunfilesTree proto
fmeum Aug 16, 2024
09bd497
WIP
fmeum Aug 16, 2024
287b3a8
Fix build
fmeum Aug 16, 2024
86ff166
Basic test setup
fmeum Aug 16, 2024
bcfc317
WIP
fmeum Aug 16, 2024
528d1be
Almost passing tests
fmeum Aug 16, 2024
13564a1
Simplify and support legacyExternalRunfiles
fmeum Aug 16, 2024
6b93b38
Fix tests
fmeum Aug 21, 2024
3f9e234
Fix tests and add docs
fmeum Aug 21, 2024
0ac5e81
Explain order
fmeum Aug 21, 2024
1e02f91
Fix test
fmeum Aug 22, 2024
80f81ac
First comments
fmeum Aug 29, 2024
cf0df2b
WIP
fmeum Aug 29, 2024
181901e
Basic runfiles test
fmeum Aug 30, 2024
eafc3e7
Key on legacyExternalRunfiles
fmeum Aug 30, 2024
880b67c
Fix collisions
fmeum Aug 30, 2024
4efa0f4
Improve collision tests
fmeum Aug 30, 2024
3806f46
Add root symlink collision test
fmeum Aug 30, 2024
8989415
Add test for `.runfile` file
fmeum Aug 30, 2024
4f321a7
Fix .runfile logic
fmeum Aug 30, 2024
4711a24
Test for cross type collision
fmeum Aug 30, 2024
06aaf17
Test postorder collisions
fmeum Aug 30, 2024
d63c480
Do not distinguish IDs by type
fmeum Aug 30, 2024
9d3560b
Store dense IDs
fmeum Aug 30, 2024
595613d
Optimize IDs
fmeum Aug 31, 2024
152a60f
Add better postorder test
fmeum Aug 31, 2024
a85be80
Test symlinks
fmeum Aug 31, 2024
0953237
Fix post order
fmeum Aug 31, 2024
8f29296
Cleanup
fmeum Aug 31, 2024
e8f854b
Add logEntryWithoutId
fmeum Sep 1, 2024
7e3df9a
Simplify ID check
fmeum Sep 1, 2024
6c245cc
Sibling test
fmeum Sep 1, 2024
5d4642e
Test with sibling layout
fmeum Sep 2, 2024
64d2c82
Encode sibling
fmeum Sep 2, 2024
a66984b
Implement support for sibling layout
fmeum Sep 2, 2024
2fb002e
Add comments and tests
fmeum Sep 2, 2024
a4413d4
More docs and uint32
fmeum Sep 2, 2024
961218c
Don't retain test runfiles trees
fmeum Sep 2, 2024
56d7a30
Fix build
fmeum Sep 2, 2024
8637fe7
Add more docs
fmeum Sep 2, 2024
b5efe8a
Inline var
fmeum Sep 2, 2024
dab1a5a
Do not flatten symlinks
fmeum Sep 15, 2024
2d87ea9
Better empty files test
fmeum Sep 17, 2024
cc18347
Update docs
fmeum Sep 17, 2024
56b6861
Simplify InputMetadataProvider
fmeum Sep 17, 2024
818f3a0
Cleanup
fmeum Sep 17, 2024
8eadcde
Add docs and test
fmeum Sep 17, 2024
8eb2cef
Simplify supporting both versions
fmeum Sep 17, 2024
786440e
Remove unused methods
fmeum Sep 17, 2024
baa6b38
Fix build
fmeum Sep 18, 2024
f8ea5b2
Address comments
fmeum Sep 19, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Key on legacyExternalRunfiles
  • Loading branch information
fmeum committed Sep 17, 2024
commit eafc3e7a5dc7edecefcb338a9b1168608be9fc13
Original file line number Diff line number Diff line change
Expand Up @@ -467,7 +467,8 @@ public void testRunfilesEmptyInput() throws Exception {
}

@Test
public void testRunfilesFilesWithMixedRoots() throws Exception {
public void testRunfilesFilesWithMixedRoots(@TestParameter boolean legacyExternalRunfiles)
throws Exception {
Artifact sourceArtifact = ActionsTestUtil.createArtifact(rootDir, "pkg/source.txt");
writeFile(sourceArtifact, "source");
Artifact genArtifact = ActionsTestUtil.createArtifact(outputDir, "other/pkg/gen.txt");
Expand All @@ -484,7 +485,14 @@ public void testRunfilesFilesWithMixedRoots() throws Exception {
PathFragment runfilesRoot = outputDir.getExecPath().getRelative("tools/foo.runfiles");
RunfilesTree runfilesTree =
createRunfilesTree(
runfilesRoot, sourceArtifact, genArtifact, externalSourceArtifact, externalGenArtifact);
runfilesRoot,
ImmutableMap.of(),
ImmutableMap.of(),
legacyExternalRunfiles,
sourceArtifact,
genArtifact,
externalSourceArtifact,
externalGenArtifact);

Spawn spawn = defaultSpawnBuilder().withInput(runfilesMiddleman).build();

Expand All @@ -507,29 +515,40 @@ public void testRunfilesFilesWithMixedRoots() throws Exception {
defaultTimeout(),
defaultSpawnResult());

closeAndAssertLog(
context,
defaultSpawnExecBuilder()
.addInputs(
File.newBuilder()
.setPath(
"bazel-out/k8-fastbuild/bin/tools/foo.runfiles/_main/other/pkg/gen.txt")
.setDigest(getDigest("gen")))
.addInputs(
File.newBuilder()
.setPath("bazel-out/k8-fastbuild/bin/tools/foo.runfiles/_main/pkg/source.txt")
.setDigest(getDigest("source")))
.addInputs(
File.newBuilder()
.setPath(
"bazel-out/k8-fastbuild/bin/tools/foo.runfiles/some_repo/other/repo_pkg/gen.txt")
.setDigest(getDigest("external_gen")))
.addInputs(
File.newBuilder()
.setPath(
"bazel-out/k8-fastbuild/bin/tools/foo.runfiles/some_repo/repo_pkg/source.txt")
.setDigest(getDigest("external_source")))
.build());
var builder = defaultSpawnExecBuilder();
if (legacyExternalRunfiles) {
builder
.addInputs(
File.newBuilder()
.setPath(
"bazel-out/k8-fastbuild/bin/tools/foo.runfiles/_main/external/some_repo/other/repo_pkg/gen.txt")
.setDigest(getDigest("external_gen")))
.addInputs(
File.newBuilder()
.setPath(
"bazel-out/k8-fastbuild/bin/tools/foo.runfiles/_main/external/some_repo/repo_pkg/source.txt")
.setDigest(getDigest("external_source")));
}
builder
.addInputs(
File.newBuilder()
.setPath("bazel-out/k8-fastbuild/bin/tools/foo.runfiles/_main/other/pkg/gen.txt")
.setDigest(getDigest("gen")))
.addInputs(
File.newBuilder()
.setPath("bazel-out/k8-fastbuild/bin/tools/foo.runfiles/_main/pkg/source.txt")
.setDigest(getDigest("source")))
.addInputs(
File.newBuilder()
.setPath(
"bazel-out/k8-fastbuild/bin/tools/foo.runfiles/some_repo/other/repo_pkg/gen.txt")
.setDigest(getDigest("external_gen")))
.addInputs(
File.newBuilder()
.setPath(
"bazel-out/k8-fastbuild/bin/tools/foo.runfiles/some_repo/repo_pkg/source.txt")
.setDigest(getDigest("external_source")));
closeAndAssertLog(context, builder.build());
}

@Test
Expand Down