diff --git a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelper.java b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelper.java index c82417b0034383..e410c0b52bc771 100644 --- a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelper.java +++ b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelper.java @@ -49,7 +49,7 @@ public final class CredentialHelper { } @VisibleForTesting - Path getPath() { + public Path getPath() { return path; } diff --git a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperProvider.java b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperProvider.java index ded7e5eab3c0d0..33dcc3dac11214 100644 --- a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperProvider.java +++ b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperProvider.java @@ -129,6 +129,23 @@ private void checkHelper(Path path) throws IOException { path.isExecutable(), "Credential helper %s is not executable", path); } + /** + * Adds a credential helper to use for all {@link URI}s matching the provided pattern, or as + * default credential helper if {@code pattern} is empty. + * + *

See {@link #add(String, Path)} for the syntax of {@code pattern}. + */ + public Builder add(Optional pattern, Path helper) throws IOException { + Preconditions.checkNotNull(pattern); + Preconditions.checkNotNull(helper); + + if (pattern.isPresent()) { + return add(pattern.get(), helper); + } else { + return add(helper); + } + } + /** * Adds a default credential helper to use for all {@link URI}s that don't specify a more * specific credential helper. diff --git a/src/main/java/com/google/devtools/build/lib/remote/BUILD b/src/main/java/com/google/devtools/build/lib/remote/BUILD index ed33a59565c956..1ed01f28639a1b 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/BUILD +++ b/src/main/java/com/google/devtools/build/lib/remote/BUILD @@ -59,6 +59,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:top_level_artifact_context", "//src/main/java/com/google/devtools/build/lib/analysis/platform:platform_utils", "//src/main/java/com/google/devtools/build/lib/authandtls", + "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper", "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/buildeventstream", "//src/main/java/com/google/devtools/build/lib/clock", @@ -102,6 +103,7 @@ java_library( "//src/main/java/com/google/devtools/common/options", "//src/main/protobuf:failure_details_java_proto", "//third_party:auth", + "//third_party:auto_value", "//third_party:caffeine", "//third_party:flogger", "//third_party:guava", diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java index 19a2b7d0a85ecc..c5f59d87114c4e 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java @@ -19,6 +19,7 @@ import build.bazel.remote.execution.v2.DigestFunction; import build.bazel.remote.execution.v2.ServerCapabilities; import com.google.auth.Credentials; +import com.google.auto.value.AutoValue; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Ascii; import com.google.common.base.Preconditions; @@ -50,6 +51,8 @@ import com.google.devtools.build.lib.authandtls.Netrc; import com.google.devtools.build.lib.authandtls.NetrcCredentials; import com.google.devtools.build.lib.authandtls.NetrcParser; +import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperEnvironment; +import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperProvider; import com.google.devtools.build.lib.bazel.repository.downloader.Downloader; import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploader; import com.google.devtools.build.lib.buildeventstream.LocalFilesArtifactUploader; @@ -78,6 +81,7 @@ import com.google.devtools.build.lib.runtime.BuildEventArtifactUploaderFactory; import com.google.devtools.build.lib.runtime.Command; import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.runtime.CommandLinePathFactory; import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutor; import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutorFactory; import com.google.devtools.build.lib.runtime.ServerBuilder; @@ -1130,4 +1134,53 @@ static Credentials newCredentials( return creds; } + + @VisibleForTesting + static CredentialHelperProvider newCredentialHelperProvider( + CredentialHelperEnvironment environment, + CommandLinePathFactory pathFactory, + List inputs) + throws IOException { + Preconditions.checkNotNull(environment); + Preconditions.checkNotNull(pathFactory); + Preconditions.checkNotNull(inputs); + + CredentialHelperProvider.Builder builder = CredentialHelperProvider.builder(); + for (String input : inputs) { + ScopedCredentialHelper helper = parseCredentialHelperFlag(environment, pathFactory, input); + builder.add(helper.getScope(), helper.getPath()); + } + return builder.build(); + } + + @VisibleForTesting + static ScopedCredentialHelper parseCredentialHelperFlag( + CredentialHelperEnvironment environment, CommandLinePathFactory pathFactory, String input) + throws IOException { + Preconditions.checkNotNull(environment); + Preconditions.checkNotNull(pathFactory); + Preconditions.checkNotNull(input); + + int pos = input.indexOf('='); + if (pos > 0) { + String scope = input.substring(0, pos); + String path = input.substring(pos + 1); + return new AutoValue_RemoteModule_ScopedCredentialHelper( + Optional.of(scope), pathFactory.create(environment.getClientEnvironment(), path)); + } + + // `input` does not specify a scope. + return new AutoValue_RemoteModule_ScopedCredentialHelper( + Optional.empty(), pathFactory.create(environment.getClientEnvironment(), input)); + } + + @VisibleForTesting + @AutoValue + static abstract class ScopedCredentialHelper { + /** Returns the scope of the credential helper (if any). */ + public abstract Optional getScope(); + + /** Returns the path of the credential helper. */ + public abstract Path getPath(); + } } diff --git a/src/test/java/com/google/devtools/build/lib/remote/BUILD b/src/test/java/com/google/devtools/build/lib/remote/BUILD index 44df625842371a..0102233157caee 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/BUILD +++ b/src/test/java/com/google/devtools/build/lib/remote/BUILD @@ -56,6 +56,7 @@ java_test( "//src/main/java/com/google/devtools/build/lib/analysis:server_directories", "//src/main/java/com/google/devtools/build/lib/analysis/platform:platform_utils", "//src/main/java/com/google/devtools/build/lib/authandtls", + "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper", "//src/main/java/com/google/devtools/build/lib/buildeventstream", "//src/main/java/com/google/devtools/build/lib/clock", "//src/main/java/com/google/devtools/build/lib/collect/nestedset", diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteModuleTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteModuleTest.java index bfc0171b6b84f6..027287570c17f8 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteModuleTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteModuleTest.java @@ -27,6 +27,7 @@ import build.bazel.remote.execution.v2.GetCapabilitiesRequest; import build.bazel.remote.execution.v2.ServerCapabilities; import com.google.auth.Credentials; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; @@ -36,6 +37,8 @@ import com.google.devtools.build.lib.analysis.config.CoreOptions; import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions; import com.google.devtools.build.lib.authandtls.BasicHttpAuthenticationEncoder; +import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperEnvironment; +import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperProvider; import com.google.devtools.build.lib.events.Reporter; import com.google.devtools.build.lib.exec.BinTools; import com.google.devtools.build.lib.exec.ExecutionOptions; @@ -47,12 +50,14 @@ import com.google.devtools.build.lib.runtime.ClientOptions; import com.google.devtools.build.lib.runtime.Command; import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.runtime.CommandLinePathFactory; import com.google.devtools.build.lib.runtime.CommonCommandOptions; import com.google.devtools.build.lib.runtime.commands.BuildCommand; import com.google.devtools.build.lib.testutil.Scratch; import com.google.devtools.build.lib.util.AbruptExitException; import com.google.devtools.build.lib.vfs.DigestHashFunction; import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import com.google.devtools.common.options.Options; import com.google.devtools.common.options.OptionsParser; @@ -65,7 +70,9 @@ import io.grpc.stub.StreamObserver; import io.grpc.util.MutableHandlerRegistry; import java.io.IOException; +import java.io.OutputStream; import java.net.URI; +import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -576,4 +583,225 @@ private static void assertRequestMetadata( assertThat(Iterables.getOnlyElement(requestMetadata.values())) .containsExactly(BasicHttpAuthenticationEncoder.encode(username, password, UTF_8)); } -} + + @Test + public void parseCredentialHelperFlag() throws Exception { + FileSystem fileSystem = new InMemoryFileSystem(DigestHashFunction.SHA256); + + Path workspace = fileSystem.getPath("/workspace"); + Path pathValue = fileSystem.getPath("/usr/local/bin"); + pathValue.createDirectoryAndParents(); + + CredentialHelperEnvironment credentialHelperEnvironment = + CredentialHelperEnvironment.newBuilder() + .setEventReporter(new Reporter(new EventBus())) + .setWorkspacePath(workspace) + .setClientEnvironment(ImmutableMap.of("PATH", pathValue.getPathString())) + .setHelperExecutionTimeout(Duration.ZERO) + .build(); + CommandLinePathFactory commandLinePathFactory = + new CommandLinePathFactory( + fileSystem, + ImmutableMap.builder() + .put("workspace", workspace) + .build()); + + // Absolute paths. + { + RemoteModule.ScopedCredentialHelper helper1 = + RemoteModule.parseCredentialHelperFlag( + credentialHelperEnvironment, commandLinePathFactory, "/absolute/path"); + assertThat(helper1.getScope().isPresent()).isFalse(); + assertThat(helper1.getPath()).isEqualTo(fileSystem.getPath("/absolute/path")); + + RemoteModule.ScopedCredentialHelper helper2 = + RemoteModule.parseCredentialHelperFlag( + credentialHelperEnvironment, commandLinePathFactory, "example.com=/absolute/path"); + assertThat(helper2.getScope().get()).isEqualTo("example.com"); + assertThat(helper2.getPath()).isEqualTo(fileSystem.getPath("/absolute/path")); + + RemoteModule.ScopedCredentialHelper helper3 = + RemoteModule.parseCredentialHelperFlag( + credentialHelperEnvironment, commandLinePathFactory, "*.example.com=/absolute/path"); + assertThat(helper3.getScope().get()).isEqualTo("*.example.com"); + assertThat(helper3.getPath()).isEqualTo(fileSystem.getPath("/absolute/path")); + } + + // Root-relative paths. + { + RemoteModule.ScopedCredentialHelper helper1 = + RemoteModule.parseCredentialHelperFlag( + credentialHelperEnvironment, commandLinePathFactory, "%workspace%/path"); + assertThat(helper1.getScope().isPresent()).isFalse(); + assertThat(helper1.getPath()).isEqualTo(workspace.getRelative("path")); + + RemoteModule.ScopedCredentialHelper helper2 = + RemoteModule.parseCredentialHelperFlag( + credentialHelperEnvironment, commandLinePathFactory, "example.com=%workspace%/path"); + assertThat(helper2.getScope().get()).isEqualTo("example.com"); + assertThat(helper2.getPath()).isEqualTo(workspace.getRelative("path")); + + RemoteModule.ScopedCredentialHelper helper3 = + RemoteModule.parseCredentialHelperFlag( + credentialHelperEnvironment, commandLinePathFactory, "*.example.com=%workspace%/path"); + assertThat(helper3.getScope().get()).isEqualTo("*.example.com"); + assertThat(helper3.getPath()).isEqualTo(workspace.getRelative("path")); + } + + // PATH lookup. + { + Path helper = createExecutable(pathValue.getRelative("foo")); + + RemoteModule.ScopedCredentialHelper helper1 = + RemoteModule.parseCredentialHelperFlag( + credentialHelperEnvironment, commandLinePathFactory, "foo"); + assertThat(helper1.getScope().isPresent()).isFalse(); + assertThat(helper1.getPath()).isEqualTo(pathValue.getRelative("foo")); + + RemoteModule.ScopedCredentialHelper helper2 = + RemoteModule.parseCredentialHelperFlag( + credentialHelperEnvironment, commandLinePathFactory, "example.com=foo"); + assertThat(helper2.getScope().get()).isEqualTo("example.com"); + assertThat(helper2.getPath()).isEqualTo(pathValue.getRelative("foo")); + + RemoteModule.ScopedCredentialHelper helper3 = + RemoteModule.parseCredentialHelperFlag( + credentialHelperEnvironment, commandLinePathFactory, "*.example.com=foo"); + assertThat(helper3.getScope().get()).isEqualTo("*.example.com"); + assertThat(helper3.getPath()).isEqualTo(pathValue.getRelative("foo")); + } + } + + @Test + public void newCredentialHelperProvider() throws Exception { + FileSystem fileSystem = new InMemoryFileSystem(DigestHashFunction.SHA256); + + Path workspace = fileSystem.getPath("/workspace"); + Path pathValue = fileSystem.getPath("/usr/local/bin"); + pathValue.createDirectoryAndParents(); + + CredentialHelperEnvironment credentialHelperEnvironment = + CredentialHelperEnvironment.newBuilder() + .setEventReporter(new Reporter(new EventBus())) + .setWorkspacePath(workspace) + .setClientEnvironment(ImmutableMap.of("PATH", pathValue.getPathString())) + .setHelperExecutionTimeout(Duration.ZERO) + .build(); + CommandLinePathFactory commandLinePathFactory = + new CommandLinePathFactory( + fileSystem, + ImmutableMap.builder() + .put("workspace", workspace) + .build()); + + Path unusedHelper = createExecutable(fileSystem, "/unused/helper"); + + Path defaultHelper = createExecutable(fileSystem, "/default/helper"); + Path exampleComHelper = createExecutable(fileSystem, "/example/com/helper"); + Path fooExampleComHelper = createExecutable(fileSystem, "/foo/example/com/helper"); + Path exampleComWildcardHelper = createExecutable(fileSystem, "/example/com/wildcard/helper"); + + Path exampleOrgHelper = createExecutable(workspace.getRelative("helpers/example-org")); + + // No helpers. + CredentialHelperProvider credentialHelperProvider1 = RemoteModule.newCredentialHelperProvider( + credentialHelperEnvironment, + commandLinePathFactory, + ImmutableList.of()); + assertThat(credentialHelperProvider1.findCredentialHelper(URI.create("https://example.com")).isPresent()).isFalse(); + assertThat(credentialHelperProvider1.findCredentialHelper(URI.create("https://foo.example.com")).isPresent()).isFalse(); + + // Default helper only. + CredentialHelperProvider credentialHelperProvider2 = RemoteModule.newCredentialHelperProvider( + credentialHelperEnvironment, + commandLinePathFactory, + ImmutableList.of(defaultHelper.getPathString())); + assertThat(credentialHelperProvider2.findCredentialHelper(URI.create("https://example.com")).get().getPath()).isEqualTo(defaultHelper); + assertThat(credentialHelperProvider2.findCredentialHelper(URI.create("https://foo.example.com")).get().getPath()).isEqualTo(defaultHelper); + + // Default and exact match. + CredentialHelperProvider credentialHelperProvider3 = RemoteModule.newCredentialHelperProvider( + credentialHelperEnvironment, + commandLinePathFactory, + ImmutableList.of( + defaultHelper.getPathString(), + "example.com=" + exampleComHelper.getPathString())); + assertThat(credentialHelperProvider3.findCredentialHelper(URI.create("https://example.com")).get().getPath()).isEqualTo(exampleComHelper); + assertThat(credentialHelperProvider3.findCredentialHelper(URI.create("https://foo.example.com")).get().getPath()).isEqualTo(defaultHelper); + + // Exact match without default. + CredentialHelperProvider credentialHelperProvider4 = RemoteModule.newCredentialHelperProvider( + credentialHelperEnvironment, + commandLinePathFactory, + ImmutableList.of("example.com=" + exampleComHelper.getPathString())); + assertThat(credentialHelperProvider4.findCredentialHelper(URI.create("https://example.com")).get().getPath()).isEqualTo(exampleComHelper); + assertThat(credentialHelperProvider4.findCredentialHelper(URI.create("https://foo.example.com")).isPresent()).isFalse(); + + // Multiple scoped helpers with default. + CredentialHelperProvider credentialHelperProvider5 = RemoteModule.newCredentialHelperProvider( + credentialHelperEnvironment, + commandLinePathFactory, + ImmutableList.of( + defaultHelper.getPathString(), + "example.com=" + exampleComHelper.getPathString(), + "*.foo.example.com=" + fooExampleComHelper.getPathString(), + "*.example.com=" + exampleComWildcardHelper.getPathString(), + "example.org=%workspace%/helpers/example-org")); + assertThat(credentialHelperProvider5.findCredentialHelper(URI.create("https://anotherdomain.com")).get().getPath()).isEqualTo(defaultHelper); + assertThat(credentialHelperProvider5.findCredentialHelper(URI.create("https://example.com")).get().getPath()).isEqualTo(exampleComHelper); + assertThat(credentialHelperProvider5.findCredentialHelper(URI.create("https://foo.example.com")).get().getPath()).isEqualTo(fooExampleComHelper); + assertThat(credentialHelperProvider5.findCredentialHelper(URI.create("https://abc.foo.example.com")).get().getPath()).isEqualTo(fooExampleComHelper); + assertThat(credentialHelperProvider5.findCredentialHelper(URI.create("https://bar.example.com")).get().getPath()).isEqualTo(exampleComWildcardHelper); + assertThat(credentialHelperProvider5.findCredentialHelper(URI.create("https://abc.bar.example.com")).get().getPath()).isEqualTo(exampleComWildcardHelper); + assertThat(credentialHelperProvider5.findCredentialHelper(URI.create("https://example.org")).get().getPath()).isEqualTo(exampleOrgHelper); + + // Helpers override. + CredentialHelperProvider credentialHelperProvider6 = RemoteModule.newCredentialHelperProvider( + credentialHelperEnvironment, + commandLinePathFactory, + ImmutableList.of( + // + unusedHelper.getPathString(), + + // + defaultHelper.getPathString(), + "example.com=" + unusedHelper.getPathString(), + "*.example.com=" + unusedHelper.getPathString(), + "example.org=" + unusedHelper.getPathString(), + "*.example.org=" + exampleOrgHelper.getPathString(), + + // + "*.example.com=" + exampleComWildcardHelper.getPathString(), + "example.org=" + exampleOrgHelper.getPathString(), + "*.foo.example.com=" + unusedHelper.getPathString(), + + // + "example.com=" + exampleComHelper.getPathString(), + "*.foo.example.com=" + fooExampleComHelper.getPathString())); + assertThat(credentialHelperProvider6.findCredentialHelper(URI.create("https://anotherdomain.com")).get().getPath()).isEqualTo(defaultHelper); + assertThat(credentialHelperProvider6.findCredentialHelper(URI.create("https://example.com")).get().getPath()).isEqualTo(exampleComHelper); + assertThat(credentialHelperProvider6.findCredentialHelper(URI.create("https://foo.example.com")).get().getPath()).isEqualTo(fooExampleComHelper); + assertThat(credentialHelperProvider6.findCredentialHelper(URI.create("https://bar.example.com")).get().getPath()).isEqualTo(exampleComWildcardHelper); + assertThat(credentialHelperProvider6.findCredentialHelper(URI.create("https://example.org")).get().getPath()).isEqualTo(exampleOrgHelper); + assertThat(credentialHelperProvider6.findCredentialHelper(URI.create("https://foo.example.org")).get().getPath()).isEqualTo(exampleOrgHelper); + } + + private static Path createExecutable(FileSystem fileSystem, String path) throws IOException { + Preconditions.checkNotNull(fileSystem); + Preconditions.checkNotNull(path); + + return createExecutable(fileSystem.getPath(path)); + } + + private static Path createExecutable(Path path) throws IOException { + Preconditions.checkNotNull(path); + + path.getParentDirectory().createDirectoryAndParents(); + try (OutputStream unused = path.getOutputStream()) { + // Nothing to do. + } + path.setExecutable(true); + + return path; + } +} \ No newline at end of file