diff --git a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/CommandHelper.java b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/CommandHelper.java index 24c6efd978b..065d3074ad3 100644 --- a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/CommandHelper.java +++ b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/CommandHelper.java @@ -34,6 +34,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import javax.annotation.Nullable; /** @@ -129,7 +130,7 @@ public CommandHelper( this.toolsRunfilesSuppliers = SkylarkList.createImmutable(toolsRunfilesBuilder.build()); ImmutableMap.Builder> labelMapBuilder = ImmutableMap.builder(); - for (Map.Entry> entry : tempLabelMap.entrySet()) { + for (Entry> entry : tempLabelMap.entrySet()) { labelMapBuilder.put(entry.getKey(), ImmutableList.copyOf(entry.getValue())); } this.labelMap = labelMapBuilder.build(); diff --git a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/ShToolchain.java b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/ShToolchain.java new file mode 100644 index 00000000000..c785da410e2 --- /dev/null +++ b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/ShToolchain.java @@ -0,0 +1,62 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** Class to work with the shell toolchain, e.g. get the shell interpreter's path. */ +public final class ShToolchain { + + /** + * Returns the shell executable's path, or an empty path if not set. + * + *

This method checks the configuration's {@link ShellConfiguration} fragment. + */ + public static PathFragment getPath(BuildConfiguration config) { + PathFragment result = PathFragment.EMPTY_FRAGMENT; + + ShellConfiguration configFragment = + (ShellConfiguration) config.getFragment(ShellConfiguration.class); + if (configFragment != null) { + PathFragment path = configFragment.getShellExecutable(); + if (path != null) { + result = path; + } + } + + return result; + } + + /** + * Returns the shell executable's path, or reports a rule error if the path is empty. + * + *

This method checks the rule's configuration's {@link ShellConfiguration} fragment for the + * shell executable's path. If null or empty, the method reports an error against the rule. + */ + public static PathFragment getPathOrError(RuleContext ctx) { + PathFragment result = getPath(ctx.getConfiguration()); + + if (result.isEmpty()) { + ctx.ruleError( + "This rule needs a shell interpreter. Use the --shell_executable= flag to specify" + + " the interpreter's path, e.g. --shell_executable=/usr/local/bin/bash"); + } + + return result; + } + + private ShToolchain() {} +} diff --git a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java index 5532b2223da..c9a5a2dddc9 100644 --- a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java +++ b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java @@ -40,10 +40,6 @@ import com.google.devtools.build.lib.actions.CommandLine; import com.google.devtools.build.lib.actions.CommandLineExpansionException; import com.google.devtools.build.lib.actions.CommandLineItemSimpleFormatter; -import com.google.devtools.build.lib.actions.CommandLines; -import com.google.devtools.build.lib.actions.CommandLines.CommandLineAndParamFileInfo; -import com.google.devtools.build.lib.actions.CommandLines.CommandLineLimits; -import com.google.devtools.build.lib.actions.CommandLines.ExpandedCommandLines; import com.google.devtools.build.lib.actions.CompositeRunfilesSupplier; import com.google.devtools.build.lib.actions.EmptyRunfilesSupplier; import com.google.devtools.build.lib.actions.ExecException; @@ -95,8 +91,8 @@ public interface ExtraActionInfoSupplier { private static final String GUID = "ebd6fce3-093e-45ee-adb6-bf513b602f0d"; - @VisibleForSerialization protected final CommandLines commandLines; - @VisibleForSerialization protected final CommandLineLimits commandLineLimits; + @VisibleForSerialization + protected final CommandLine argv; private final boolean executeUnconditionally; private final boolean isShellCommand; @@ -107,7 +103,6 @@ public interface ExtraActionInfoSupplier { private final ImmutableMap executionInfo; private final ExtraActionInfoSupplier extraActionInfoSupplier; - private final Artifact primaryOutput; /** * Constructs a SpawnAction using direct initialization arguments. @@ -119,12 +114,11 @@ public interface ExtraActionInfoSupplier { * @param inputs the set of all files potentially read by this action; must not be subsequently * modified. * @param outputs the set of all files written by this action; must not be subsequently modified. - * @param primaryOutput the primary output of this action * @param resourceSet the resources consumed by executing this Action * @param env the action environment - * @param commandLines the command lines to execute. This includes the main argv vector and any - * param file-backed command lines. - * @param commandLineLimits the command line limits, from the build configuration + * @param argv the command line to execute. This is merely a list of options to the executable, + * and is uninterpreted by the build tool for the purposes of dependency checking; typically + * it may include the names of input and output files, but this is not necessary. * @param isShellCommand Whether the command line represents a shell command with the given shell * executable. This is used to give better error messages. * @param progressMessage the message printed during the progression of the build. @@ -136,10 +130,8 @@ public SpawnAction( Iterable tools, Iterable inputs, Iterable outputs, - Artifact primaryOutput, ResourceSet resourceSet, - CommandLines commandLines, - CommandLineLimits commandLineLimits, + CommandLine argv, boolean isShellCommand, ActionEnvironment env, CharSequence progressMessage, @@ -149,10 +141,8 @@ public SpawnAction( tools, inputs, outputs, - primaryOutput, resourceSet, - commandLines, - commandLineLimits, + argv, isShellCommand, env, ImmutableMap.of(), @@ -174,13 +164,13 @@ public SpawnAction( * @param inputs the set of all files potentially read by this action; must not be subsequently * modified * @param outputs the set of all files written by this action; must not be subsequently modified. - * @param primaryOutput the primary output of this action * @param resourceSet the resources consumed by executing this Action * @param env the action's environment * @param executionInfo out-of-band information for scheduling the spawn - * @param commandLines the command lines to execute. This includes the main argv vector and any - * param file-backed command lines. - * @param commandLineLimits the command line limits, from the build configuration + * @param argv the argv array (including argv[0]) of arguments to pass. This is merely a list of + * options to the executable, and is uninterpreted by the build tool for the purposes of + * dependency checking; typically it may include the names of input and output files, but this + * is not necessary. * @param isShellCommand Whether the command line represents a shell command with the given shell * executable. This is used to give better error messages. * @param progressMessage the message printed during the progression of the build @@ -192,10 +182,8 @@ public SpawnAction( Iterable tools, Iterable inputs, Iterable outputs, - Artifact primaryOutput, ResourceSet resourceSet, - CommandLines commandLines, - CommandLineLimits commandLineLimits, + CommandLine argv, boolean isShellCommand, ActionEnvironment env, ImmutableMap executionInfo, @@ -205,11 +193,9 @@ public SpawnAction( boolean executeUnconditionally, ExtraActionInfoSupplier extraActionInfoSupplier) { super(owner, tools, inputs, runfilesSupplier, outputs, env); - this.primaryOutput = primaryOutput; this.resourceSet = resourceSet; this.executionInfo = executionInfo; - this.commandLines = commandLines; - this.commandLineLimits = commandLineLimits; + this.argv = argv; this.isShellCommand = isShellCommand; this.progressMessage = progressMessage; this.mnemonic = mnemonic; @@ -217,15 +203,10 @@ public SpawnAction( this.extraActionInfoSupplier = extraActionInfoSupplier; } - @Override - public Artifact getPrimaryOutput() { - return primaryOutput; - } - @Override @VisibleForTesting public List getArguments() throws CommandLineExpansionException { - return ImmutableList.copyOf(commandLines.allArguments()); + return ImmutableList.copyOf(argv.arguments()); } @Override @@ -233,6 +214,10 @@ public SkylarkList getSkylarkArgv() throws CommandLineExpansionException return SkylarkList.createImmutable(getArguments()); } + protected CommandLine getCommandLine() { + return argv; + } + @Override @VisibleForTesting public Iterable getPossibleInputsForTesting() { @@ -242,13 +227,13 @@ public Iterable getPossibleInputsForTesting() { /** Returns command argument, argv[0]. */ @VisibleForTesting public String getCommandFilename() throws CommandLineExpansionException { - return Iterables.getFirst(getArguments(), null); + return Iterables.getFirst(argv.arguments(), null); } /** Returns the (immutable) list of arguments, excluding the command name, argv[0]. */ @VisibleForTesting public List getRemainingArguments() throws CommandLineExpansionException { - return ImmutableList.copyOf(Iterables.skip(getArguments(), 1)); + return ImmutableList.copyOf(Iterables.skip(argv.arguments(), 1)); } @VisibleForTesting @@ -273,7 +258,7 @@ public boolean executeUnconditionally() { */ protected List internalExecute(ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException, CommandLineExpansionException { - Spawn spawn = getSpawn(actionExecutionContext); + Spawn spawn = getSpawn(actionExecutionContext.getClientEnv()); return getContext(actionExecutionContext, spawn).exec(spawn, actionExecutionContext); } @@ -296,7 +281,7 @@ public ActionResult execute(ActionExecutionContext actionExecutionContext) failMessage = "error executing shell command: " + "'" - + truncate(Iterables.get(getArguments(), 2), 200) + + truncate(Iterables.get(argv.arguments(), 2), 200) + "'"; } catch (CommandLineExpansionException commandLineExpansionException) { failMessage = @@ -329,37 +314,25 @@ private static String truncate(String s, int maxLen) { * *

This method is final, as it is merely a shorthand use of the generic way to obtain a spawn, * which also depends on the client environment. Subclasses that which to override the way to get - * a spawn should override {@link #getSpawn()} instead. + * a spawn should override {@link #getSpawn(Map)} instead. */ public final Spawn getSpawn() throws CommandLineExpansionException { - return new ActionSpawn(commandLines.allArguments(), null, ImmutableList.of()); + return getSpawn(null); } /** * Return a spawn that is representative of the command that this Action will execute in the given * client environment. */ - public Spawn getSpawn(ActionExecutionContext actionExecutionContext) - throws CommandLineExpansionException { - return getSpawn( - actionExecutionContext.getArtifactExpander(), actionExecutionContext.getClientEnv()); - } - - Spawn getSpawn(ArtifactExpander artifactExpander, Map clientEnv) - throws CommandLineExpansionException { - ExpandedCommandLines expandedCommandLines = - commandLines.expand(artifactExpander, getPrimaryOutput().getExecPath(), commandLineLimits); - return new ActionSpawn( - ImmutableList.copyOf(expandedCommandLines.arguments()), - clientEnv, - expandedCommandLines.getParamFiles()); + public Spawn getSpawn(Map clientEnv) throws CommandLineExpansionException { + return new ActionSpawn(ImmutableList.copyOf(argv.arguments()), clientEnv); } @Override protected void computeKey(ActionKeyContext actionKeyContext, Fingerprint fp) throws CommandLineExpansionException { fp.addString(GUID); - commandLines.addToFingerprint(actionKeyContext, fp); + argv.addToFingerprint(actionKeyContext, fp); fp.addString(getMnemonic()); // We don't need the toolManifests here, because they are a subset of the inputManifests by // definition and the output of an action shouldn't change whether something is considered a @@ -393,7 +366,7 @@ public String describeKey() { message.append('\n'); } try { - for (String argument : ShellEscaper.escapeAll(getArguments())) { + for (String argument : ShellEscaper.escapeAll(argv.arguments())) { message.append(" Argument: "); message.append(argument); message.append('\n'); @@ -482,10 +455,11 @@ protected SpawnActionContext getContext( return actionExecutionContext.getSpawnActionContext(spawn); } - /** A spawn instance that is tied to a specific SpawnAction. */ - private class ActionSpawn extends BaseSpawn { + /** + * A spawn instance that is tied to a specific SpawnAction. + */ + public class ActionSpawn extends BaseSpawn { - private final ImmutableList inputs; private final ImmutableList filesets; private final ImmutableMap effectiveEnvironment; @@ -495,10 +469,7 @@ private class ActionSpawn extends BaseSpawn { *

Subclasses of ActionSpawn may subclass in order to provide action-specific values for * environment variables or action inputs. */ - private ActionSpawn( - ImmutableList arguments, - Map clientEnv, - Iterable additionalInputs) { + protected ActionSpawn(ImmutableList arguments, Map clientEnv) { super( arguments, ImmutableMap.of(), @@ -506,19 +477,13 @@ private ActionSpawn( SpawnAction.this.getRunfilesSupplier(), SpawnAction.this, resourceSet); - ImmutableList.Builder inputs = ImmutableList.builder(); - ImmutableList.Builder filesets = ImmutableList.builder(); - ImmutableList manifests = getRunfilesSupplier().getManifests(); + ImmutableList.Builder builder = ImmutableList.builder(); for (Artifact input : getInputs()) { if (input.isFileset()) { - filesets.add(input); - } else if (!manifests.contains(input)) { - inputs.add(input); + builder.add(input); } } - inputs.addAll(additionalInputs); - this.filesets = filesets.build(); - this.inputs = inputs.build(); + filesets = builder.build(); LinkedHashMap env = new LinkedHashMap<>(SpawnAction.this.getEnvironment()); if (clientEnv != null) { for (String var : SpawnAction.this.getClientEnvironmentVariables()) { @@ -546,7 +511,20 @@ public ImmutableList getFilesetManifests() { @Override @SuppressWarnings("unchecked") public Iterable getInputFiles() { - return inputs; + Iterable inputs = getInputs(); + ImmutableList manifests = getRunfilesSupplier().getManifests(); + if (filesets.isEmpty() && manifests.isEmpty()) { + return inputs; + } + // TODO(buchgr): These optimizations shouldn't exists. Instead getInputFiles() should + // directly return a NestedSet. In order to do this, rules need to be updated to + // provide inputs as nestedsets and store manifest files and filesets separately. + if (inputs instanceof NestedSet) { + return new FilesetAndManifestFilteringNestedSetView<>( + (NestedSet) inputs, filesets, manifests); + } else { + return new FilesetAndManifestFilteringIterable<>(inputs, filesets, manifests); + } } } @@ -597,6 +575,17 @@ public Iterator iterator() { */ public static class Builder { + private static class CommandLineAndParamFileInfo { + private final CommandLine commandLine; + @Nullable private final ParamFileInfo paramFileInfo; + + private CommandLineAndParamFileInfo( + CommandLine commandLine, @Nullable ParamFileInfo paramFileInfo) { + this.commandLine = commandLine; + this.paramFileInfo = paramFileInfo; + } + } + private final NestedSetBuilder toolsBuilder = NestedSetBuilder.stableOrder(); private final NestedSetBuilder inputsBuilder = NestedSetBuilder.stableOrder(); private final List outputs = new ArrayList<>(); @@ -641,7 +630,7 @@ public Builder(Builder other) { this.executableArgs = (other.executableArgs != null) ? Lists.newArrayList(other.executableArgs) : null; - this.commandLines = new ArrayList<>(other.commandLines); + this.commandLines = Lists.newArrayList(other.commandLines); this.progressMessage = other.progressMessage; this.mnemonic = other.mnemonic; } @@ -672,65 +661,25 @@ public Action[] build(ActionConstructionContext context) { @VisibleForTesting @CheckReturnValue public Action[] build(ActionOwner owner, AnalysisEnvironment analysisEnvironment, BuildConfiguration configuration) { - final Action[] actions; - final CommandLines commandLines; - ImmutableList executableArgs = buildExecutableArgs(); - if (configuration.deferParamFiles()) { - actions = new Action[1]; - CommandLines.Builder result = CommandLines.builder(); - result.addCommandLine(CommandLine.of(executableArgs)); - for (CommandLineAndParamFileInfo pair : this.commandLines) { - result.addCommandLine(pair); - } - commandLines = result.build(); - } else { - List paramFileActions = new ArrayList<>(this.commandLines.size()); - CommandLine commandLine = - buildCommandLinesAndParamFileActions( - owner, - analysisEnvironment, - configuration, - executableArgs, - this.commandLines, - paramFileActions); - commandLines = CommandLines.of(commandLine); - actions = new Action[1 + paramFileActions.size()]; - for (int i = 0; i < paramFileActions.size(); ++i) { - actions[i + 1] = paramFileActions.get(i); - } - } - ActionEnvironment env = - useDefaultShellEnvironment - ? configuration.getActionEnvironment() - : ActionEnvironment.create(this.environment); + List paramFileActions = new ArrayList<>(commandLines.size()); + CommandLine actualCommandLine = + buildCommandLine(owner, analysisEnvironment, configuration, paramFileActions); + Action[] actions = new Action[1 + paramFileActions.size()]; Action spawnAction = - buildSpawnAction(owner, commandLines, configuration.getCommandLineLimits(), env); + buildSpawnAction(owner, actualCommandLine, configuration.getActionEnvironment()); actions[0] = spawnAction; - return actions; - } - - @CheckReturnValue - SpawnAction buildForActionTemplate(ActionOwner owner) { - CommandLines.Builder result = CommandLines.builder(); - ImmutableList executableArgs = buildExecutableArgs(); - result.addCommandLine(CommandLine.of(executableArgs)); - for (CommandLineAndParamFileInfo pair : commandLines) { - result.addCommandLine(pair.commandLine); + for (int i = 0; i < paramFileActions.size(); ++i) { + actions[i + 1] = paramFileActions.get(i); } - return buildSpawnAction( - owner, - result.build(), - CommandLineLimits.UNLIMITED, - ActionEnvironment.create(this.environment)); + return actions; } - private CommandLine buildCommandLinesAndParamFileActions( + private CommandLine buildCommandLine( ActionOwner owner, AnalysisEnvironment analysisEnvironment, BuildConfiguration configuration, - ImmutableList executableArgs, - List commandLines, List paramFileActions) { + ImmutableList executableArgs = buildExecutableArgs(); boolean hasConditionalParamFile = commandLines.stream().anyMatch(c -> c.paramFileInfo != null && !c.paramFileInfo.always()); boolean spillToParamFiles = false; @@ -740,7 +689,7 @@ private CommandLine buildCommandLinesAndParamFileActions( totalLen += getCommandLineSize(commandLineAndParamFileInfo.commandLine); } // To reduce implementation complexity we either spill all or none of the param files. - spillToParamFiles = totalLen >= configuration.getCommandLineLimits().maxLength; + spillToParamFiles = totalLen > configuration.getMinParamFileSize(); } // We a name based on the output, starting at -2.params // and then incrementing @@ -803,12 +752,18 @@ private static int getParamFileSize(Iterable args) { * *

This method makes a copy of all the collections, so it is safe to reuse the builder after * this method returns. + * + *

This method is invoked by {@link SpawnActionTemplate} in the execution phase. It is + * important that analysis-phase objects (RuleContext, Configuration, etc.) are not directly + * referenced in this function to prevent them from bleeding into the execution phase. + * + * @param owner the {@link ActionOwner} for the SpawnAction + * @param configEnv the config's action environment to use. May be null if not used. + * @return the SpawnAction and any actions required by it, with the first item always being the + * SpawnAction itself. */ - private SpawnAction buildSpawnAction( - ActionOwner owner, - CommandLines commandLines, - CommandLineLimits commandLineLimits, - ActionEnvironment env) { + SpawnAction buildSpawnAction( + ActionOwner owner, CommandLine commandLine, @Nullable ActionEnvironment configEnv) { NestedSet tools = toolsBuilder.build(); // Tools are by definition a subset of the inputs, so make sure they're present there, too. @@ -818,6 +773,13 @@ private SpawnAction buildSpawnAction( .addTransitive(tools) .build(); + ActionEnvironment env; + if (useDefaultShellEnvironment) { + env = Preconditions.checkNotNull(configEnv); + } else { + env = ActionEnvironment.create(this.environment); + } + if (disableSandboxing) { ImmutableMap.Builder builder = ImmutableMap.builder(); builder.putAll(executionInfo); @@ -830,10 +792,8 @@ private SpawnAction buildSpawnAction( tools, inputsAndTools, ImmutableList.copyOf(outputs), - outputs.get(0), resourceSet, - commandLines, - commandLineLimits, + commandLine, isShellCommand, env, ImmutableMap.copyOf(executionInfo), @@ -843,16 +803,29 @@ private SpawnAction buildSpawnAction( mnemonic); } + /** + * Builds the command line, forcing no params file. + * + *

This method is invoked by {@link SpawnActionTemplate} in the execution phase. + */ + CommandLine buildCommandLineWithoutParamsFiles() { + SpawnActionCommandLine.Builder result = new SpawnActionCommandLine.Builder(); + ImmutableList executableArgs = buildExecutableArgs(); + result.addExecutableArguments(executableArgs); + for (CommandLineAndParamFileInfo commandLineAndParamFileInfo : commandLines) { + result.addCommandLine(commandLineAndParamFileInfo.commandLine); + } + return result.build(); + } + /** Creates a SpawnAction. */ protected SpawnAction createSpawnAction( ActionOwner owner, NestedSet tools, NestedSet inputsAndTools, ImmutableList outputs, - Artifact primaryOutput, ResourceSet resourceSet, - CommandLines commandLines, - CommandLineLimits commandLineLimits, + CommandLine actualCommandLine, boolean isShellCommand, ActionEnvironment env, ImmutableMap executionInfo, @@ -864,10 +837,8 @@ protected SpawnAction createSpawnAction( tools, inputsAndTools, outputs, - primaryOutput, resourceSet, - commandLines, - commandLineLimits, + actualCommandLine, isShellCommand, env, executionInfo, diff --git a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkActionFactory.java b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkActionFactory.java index 0985afb108d..4f09dde901d 100644 --- a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkActionFactory.java +++ b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkActionFactory.java @@ -805,7 +805,7 @@ private void registerSpawnAction( } if (executionRequirementsUnchecked != Runtime.NONE) { builder.setExecutionInfo( - TargetUtils.filter( + ImmutableMap.copyOf( SkylarkDict.castSkylarkDictOrNoneToDict( executionRequirementsUnchecked, String.class, diff --git a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/test/TestActionBuilder.java b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/test/TestActionBuilder.java index 936ebd77f2a..5299d9a8b61 100644 --- a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/test/TestActionBuilder.java +++ b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/test/TestActionBuilder.java @@ -39,6 +39,7 @@ import com.google.devtools.build.lib.packages.TargetUtils; import com.google.devtools.build.lib.packages.TestSize; import com.google.devtools.build.lib.packages.TestTimeout; +import com.google.devtools.build.lib.syntax.Type; import com.google.devtools.build.lib.util.Pair; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.common.options.EnumConverter; @@ -274,6 +275,11 @@ private TestParams createTestAction(int shards) { List results = Lists.newArrayListWithCapacity(runsPerTest * shardRuns); ImmutableList.Builder coverageArtifacts = ImmutableList.builder(); + boolean useTestRunner = false; + if (ruleContext.attributes().has("use_testrunner", Type.BOOLEAN)) { + useTestRunner = ruleContext.attributes().get("use_testrunner", Type.BOOLEAN); + } + for (int run = 0; run < runsPerTest; run++) { // Use a 1-based index for user friendliness. String testRunDir = @@ -318,7 +324,8 @@ private TestParams createTestAction(int shards) { run, config, ruleContext.getWorkspaceName(), - shExecutable)); + shExecutable, + useTestRunner)); results.add(cacheStatus); } } diff --git a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/test/TestRunnerAction.java b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/test/TestRunnerAction.java index 273245c76a4..eb754bdbf4d 100644 --- a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/test/TestRunnerAction.java +++ b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/analysis/test/TestRunnerAction.java @@ -58,9 +58,9 @@ import javax.annotation.Nullable; /** - * An Action representing a test with the associated environment (runfiles, environment variables, - * test result, etc). It consumes test executable and runfiles artifacts and produces test result - * and test status artifacts. + * An Action representing a test with the associated environment (runfiles, + * environment variables, test result, etc). It consumes test executable and + * runfiles artifacts and produces test result and test status artifacts. */ // Not final so that we can mock it in tests. public class TestRunnerAction extends AbstractAction implements NotifyOnActionCacheHit { @@ -88,7 +88,8 @@ public class TestRunnerAction extends AbstractAction implements NotifyOnActionCa private final PathFragment undeclaredOutputsManifestPath; private final PathFragment undeclaredOutputsAnnotationsPath; private final PathFragment xmlOutputPath; - @Nullable private final PathFragment testShard; + @Nullable + private final PathFragment testShard; private final PathFragment testExitSafe; private final PathFragment testStderr; private final PathFragment testInfrastructureFailure; @@ -150,12 +151,10 @@ private static ImmutableList list(Artifact... artifacts) { boolean useTestRunner) { super( owner, - /*tools=*/ ImmutableList.of(), inputs, // Note that this action only cares about the runfiles, not the mapping. new RunfilesSupplierImpl(PathFragment.create("runfiles"), executionSettings.getRunfiles()), - list(testLog, cacheStatus, coverageArtifact), - configuration.getActionEnvironment()); + list(testLog, cacheStatus, coverageArtifact)); Preconditions.checkState((collectCoverageScript == null) == (coverageArtifact == null)); this.testSetupScript = testSetupScript; this.collectCoverageScript = collectCoverageScript; @@ -173,12 +172,13 @@ private static ImmutableList list(Artifact... artifacts) { this.baseDir = cacheStatus.getExecPath().getParentDirectory(); int totalShards = executionSettings.getTotalShards(); - Preconditions.checkState( - (totalShards == 0 && shardNum == 0) - || (totalShards > 0 && 0 <= shardNum && shardNum < totalShards)); + Preconditions.checkState((totalShards == 0 && shardNum == 0) + || (totalShards > 0 && 0 <= shardNum && shardNum < totalShards)); this.testExitSafe = baseDir.getChild("test.exited_prematurely"); // testShard Path should be set only if sharding is enabled. - this.testShard = totalShards > 1 ? baseDir.getChild("test.shard") : null; + this.testShard = totalShards > 1 + ? baseDir.getChild("test.shard") + : null; this.xmlOutputPath = baseDir.getChild("test.xml"); this.testWarningsPath = baseDir.getChild("test.warnings"); this.unusedRunfilesLogPath = baseDir.getChild("test.unused_runfiles_log"); @@ -198,10 +198,9 @@ private static ImmutableList list(Artifact... artifacts) { this.extraTestEnv = ImmutableMap.copyOf(extraTestEnv); this.requiredClientEnvVariables = - ImmutableIterable.from( - Iterables.concat( - configuration.getActionEnvironment().getInheritedEnv(), - configuration.getTestActionEnvironment().getInheritedEnv())); + ImmutableIterable.from(Iterables.concat( + configuration.getActionEnvironment().getInheritedEnv(), + configuration.getTestActionEnvironment().getInheritedEnv())); } public BuildConfiguration getConfiguration() { @@ -259,38 +258,28 @@ public ImmutableList> getTestOutputsMapping(Path execRoot) { builder.add(Pair.of(TestFileNameConstants.SPLIT_LOGS, resolvedPaths.getSplitLogsPath())); } if (resolvedPaths.getTestWarningsPath().exists()) { - builder.add( - Pair.of(TestFileNameConstants.TEST_WARNINGS, resolvedPaths.getTestWarningsPath())); + builder.add(Pair.of(TestFileNameConstants.TEST_WARNINGS, + resolvedPaths.getTestWarningsPath())); } if (resolvedPaths.getUndeclaredOutputsZipPath().exists()) { - builder.add( - Pair.of( - TestFileNameConstants.UNDECLARED_OUTPUTS_ZIP, - resolvedPaths.getUndeclaredOutputsZipPath())); + builder.add(Pair.of(TestFileNameConstants.UNDECLARED_OUTPUTS_ZIP, + resolvedPaths.getUndeclaredOutputsZipPath())); } if (resolvedPaths.getUndeclaredOutputsManifestPath().exists()) { - builder.add( - Pair.of( - TestFileNameConstants.UNDECLARED_OUTPUTS_MANIFEST, - resolvedPaths.getUndeclaredOutputsManifestPath())); + builder.add(Pair.of(TestFileNameConstants.UNDECLARED_OUTPUTS_MANIFEST, + resolvedPaths.getUndeclaredOutputsManifestPath())); } if (resolvedPaths.getUndeclaredOutputsAnnotationsPath().exists()) { - builder.add( - Pair.of( - TestFileNameConstants.UNDECLARED_OUTPUTS_ANNOTATIONS, - resolvedPaths.getUndeclaredOutputsAnnotationsPath())); + builder.add(Pair.of(TestFileNameConstants.UNDECLARED_OUTPUTS_ANNOTATIONS, + resolvedPaths.getUndeclaredOutputsAnnotationsPath())); } if (resolvedPaths.getUnusedRunfilesLogPath().exists()) { - builder.add( - Pair.of( - TestFileNameConstants.UNUSED_RUNFILES_LOG, - resolvedPaths.getUnusedRunfilesLogPath())); + builder.add(Pair.of(TestFileNameConstants.UNUSED_RUNFILES_LOG, + resolvedPaths.getUnusedRunfilesLogPath())); } if (resolvedPaths.getInfrastructureFailureFile().exists()) { - builder.add( - Pair.of( - TestFileNameConstants.TEST_INFRASTRUCTURE_FAILURE, - resolvedPaths.getInfrastructureFailureFile())); + builder.add(Pair.of(TestFileNameConstants.TEST_INFRASTRUCTURE_FAILURE, + resolvedPaths.getInfrastructureFailureFile())); } } return builder.build(); @@ -337,14 +326,18 @@ public boolean isVolatile() { return true; } - /** Saves cache status to disk. */ + /** + * Saves cache status to disk. + */ public void saveCacheStatus(TestResultData data) throws IOException { try (OutputStream out = cacheStatus.getPath().getOutputStream()) { data.writeTo(out); } } - /** Returns the cache from disk, or null if the file doesn't exist or if there is an error. */ + /** + * Returns the cache from disk, or null if the file doesn't exist or if there is an error. + */ @Nullable private TestResultData readCacheStatus() { try (InputStream in = cacheStatus.getPath().getInputStream()) { @@ -386,9 +379,9 @@ static boolean canBeCached( } /** - * Returns whether caching has been deemed safe by looking at the previous test run (for local - * caching). If the previous run is not present, return "true" here, as remote execution caching - * should be safe. + * Returns whether caching has been deemed safe by looking at the previous test run + * (for local caching). If the previous run is not present, return "true" here, as + * remote execution caching should be safe. */ public boolean shouldCacheResult() { return !executeUnconditionally(); @@ -398,12 +391,9 @@ public boolean shouldCacheResult() { public void actionCacheHit(ActionCachedContext executor) { unconditionalExecution = null; try { - executor - .getEventBus() - .post( - executor - .getContext(TestActionContext.class) - .newCachedTestResult(executor.getExecRoot(), this, readCacheStatus())); + executor.getEventBus().post( + executor.getContext(TestActionContext.class).newCachedTestResult( + executor.getExecRoot(), this, readCacheStatus())); } catch (IOException e) { LoggingUtil.logToRemote(Level.WARNING, "Failed creating cached protocol buffer", e); } @@ -518,6 +508,11 @@ public void setupEnvVariables(Map env, Duration timeout) { } env.put("XML_OUTPUT_FILE", getXmlOutputPath().getPathString()); + if (!isEnableRunfiles()) { + // If runfiles are disabled, tell remote-runtest.sh/local-runtest.sh about that. + env.put("RUNFILES_MANIFEST_ONLY", "1"); + } + if (isCoverageMode()) { // Instruct remote-runtest.sh/local-runtest.sh not to cd into the runfiles directory. // TODO(ulfjack): Find a way to avoid setting this variable. @@ -534,8 +529,8 @@ public void setupEnvVariables(Map env, Duration timeout) { env.put("NEW_JAVA_COVERAGE_IMPL", "released"); } else { // This value ("True") should have told lcov_merger whether it should use the old or the new - // java coverage implementation. Due to several failed attempts at submitting the new - // implementation, this value will be treated still as the old implementation. This + // java coverage implementation. Due to several failed attempts at submitting the new + // implementation, this value will be treated still as the old implementation. This // environment variable must be set to a value recognized by lcov_merger. env.put("NEW_JAVA_COVERAGE_IMPL", "True"); } @@ -543,26 +538,26 @@ public void setupEnvVariables(Map env, Duration timeout) { } /** - * Gets the test name in a user-friendly format. Will generally include the target name and - * run/shard numbers, if applicable. + * Gets the test name in a user-friendly format. + * Will generally include the target name and run/shard numbers, if applicable. */ public String getTestName() { String suffix = getTestSuffix(); String label = Label.print(getOwner().getLabel()); - return suffix.isEmpty() ? label : label + " " + suffix; + return suffix.isEmpty() ? label : label + " " + suffix; } /** - * Gets the test suffix in a user-friendly format, eg "(shard 1 of 7)". Will include the target - * name and run/shard numbers, if applicable. + * Gets the test suffix in a user-friendly format, eg "(shard 1 of 7)". + * Will include the target name and run/shard numbers, if applicable. */ public String getTestSuffix() { int totalShards = executionSettings.getTotalShards(); // Use a 1-based index for user friendliness. int runsPerTest = testConfiguration.getRunsPerTestForLabel(getOwner().getLabel()); if (totalShards > 1 && runsPerTest > 1) { - return String.format( - "(shard %d of %d, run %d of %d)", shardNum + 1, totalShards, runNumber + 1, runsPerTest); + return String.format("(shard %d of %d, run %d of %d)", shardNum + 1, totalShards, + runNumber + 1, runsPerTest); } else if (totalShards > 1) { return String.format("(shard %d of %d)", shardNum + 1, totalShards); } else if (runsPerTest > 1) { @@ -576,7 +571,9 @@ public Artifact getTestLog() { return testLog; } - /** Returns all environment variables which must be set in order to run this test. */ + /** + * Returns all environment variables which must be set in order to run this test. + */ public Map getExtraTestEnv() { return extraTestEnv; } @@ -610,12 +607,16 @@ public PathFragment getUndeclaredOutputsDir() { return undeclaredOutputsDir; } - /** @return path to the optional zip file of undeclared test outputs. */ + /** + * @return path to the optional zip file of undeclared test outputs. + */ public PathFragment getUndeclaredOutputsZipPath() { return undeclaredOutputsZipPath; } - /** @return path to the undeclared output manifest file. */ + /** + * @return path to the undeclared output manifest file. + */ public PathFragment getUndeclaredOutputsManifestPath() { return undeclaredOutputsManifestPath; } @@ -624,7 +625,9 @@ public PathFragment getUndeclaredOutputsAnnotationsDir() { return undeclaredOutputsAnnotationsDir; } - /** @return path to the undeclared output annotations file. */ + /** + * @return path to the undeclared output annotations file. + */ public PathFragment getUndeclaredOutputsAnnotationsPath() { return undeclaredOutputsAnnotationsPath; } @@ -641,19 +644,21 @@ public PathFragment getInfrastructureFailureFile() { return testInfrastructureFailure; } - /** @return path to the optionally created XML output file created by the test. */ + /** + * @return path to the optionally created XML output file created by the test. + */ public PathFragment getXmlOutputPath() { return xmlOutputPath; } - /** @return coverage data artifact or null if code coverage was not requested. */ - @Nullable - public Artifact getCoverageData() { + /** + * @return coverage data artifact or null if code coverage was not requested. + */ + @Nullable public Artifact getCoverageData() { return coverageData; } - @Nullable - public Artifact getCoverageManifest() { + @Nullable public Artifact getCoverageManifest() { return getExecutionSettings().getInstrumentedFileManifest(); } @@ -694,19 +699,24 @@ public boolean isSharded() { } /** - * @return the shard number for this action. If getTotalShards() > 0, must be >= 0 and < - * getTotalShards(). Otherwise, must be 0. + * @return the shard number for this action. + * If getTotalShards() > 0, must be >= 0 and < getTotalShards(). + * Otherwise, must be 0. */ public int getShardNum() { return shardNum; } - /** @return run number. */ + /** + * @return run number. + */ public int getRunNumber() { return runNumber; } - /** @return the workspace name. */ + /** + * @return the workspace name. + */ public String getRunfilesPrefix() { return workspaceName; } @@ -743,8 +753,7 @@ public Artifact getTestSetupScript() { return testSetupScript; } - @Nullable - public Artifact getCollectCoverageScript() { + @Nullable public Artifact getCollectCoverageScript() { return collectCoverageScript; } @@ -760,7 +769,9 @@ public boolean isEnableRunfiles() { return configuration.runfilesEnabled(); } - /** The same set of paths as the parent test action, resolved against a given exec root. */ + /** + * The same set of paths as the parent test action, resolved against a given exec root. + */ public final class ResolvedPaths { private final Path execRoot; @@ -796,32 +807,44 @@ public Path getUnusedRunfilesLogPath() { return getPath(unusedRunfilesLogPath); } - /** @return path to the directory containing the split logs (raw and proto file). */ + /** + * @return path to the directory containing the split logs (raw and proto file). + */ public Path getSplitLogsDir() { return getPath(splitLogsDir); } - /** @return path to the optional zip file of undeclared test outputs. */ + /** + * @return path to the optional zip file of undeclared test outputs. + */ public Path getUndeclaredOutputsZipPath() { return getPath(undeclaredOutputsZipPath); } - /** @return path to the directory to hold undeclared test outputs. */ + /** + * @return path to the directory to hold undeclared test outputs. + */ public Path getUndeclaredOutputsDir() { return getPath(undeclaredOutputsDir); } - /** @return path to the directory to hold undeclared output annotations parts. */ + /** + * @return path to the directory to hold undeclared output annotations parts. + */ public Path getUndeclaredOutputsAnnotationsDir() { return getPath(undeclaredOutputsAnnotationsDir); } - /** @return path to the undeclared output manifest file. */ + /** + * @return path to the undeclared output manifest file. + */ public Path getUndeclaredOutputsManifestPath() { return getPath(undeclaredOutputsManifestPath); } - /** @return path to the undeclared output annotations file. */ + /** + * @return path to the undeclared output annotations file. + */ public Path getUndeclaredOutputsAnnotationsPath() { return getPath(undeclaredOutputsAnnotationsPath); } @@ -839,7 +862,9 @@ public Path getInfrastructureFailureFile() { return getPath(testInfrastructureFailure); } - /** @return path to the optionally created XML output file created by the test. */ + /** + * @return path to the optionally created XML output file created by the test. + */ public Path getXmlOutputPath() { return getPath(xmlOutputPath); } diff --git a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaSemantics.java b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaSemantics.java index 4efc58365bc..13879ef5347 100644 --- a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaSemantics.java +++ b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaSemantics.java @@ -514,6 +514,7 @@ private static NestedSet getRuntimeJarsForTargets(TransitiveInfoCollec // dep.getProvider(JavaCompilationArgsProvider.class).getRecursiveJavaCompilationArgs(), // so we reuse the logic within JavaCompilationArgs to handle both scenarios. return JavaCompilationArgsProvider.legacyFromTargets(ImmutableList.copyOf(deps)) + .getRecursiveJavaCompilationArgs() .getRuntimeJars(); } diff --git a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java index e8137af4725..55e3b957751 100644 --- a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java +++ b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java @@ -19,7 +19,6 @@ import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.actions.ParamFileInfo; import com.google.devtools.build.lib.actions.ParameterFile; import com.google.devtools.build.lib.analysis.FilesToRunProvider; import com.google.devtools.build.lib.analysis.RuleContext; @@ -30,6 +29,7 @@ import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; import com.google.devtools.build.lib.analysis.actions.LauncherFileWriteAction; import com.google.devtools.build.lib.analysis.actions.LauncherFileWriteAction.LaunchInfo; +import com.google.devtools.build.lib.analysis.actions.ParameterFileWriteAction; import com.google.devtools.build.lib.analysis.actions.SpawnAction; import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction; import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction.Substitution; @@ -290,20 +290,32 @@ private static void createPythonZipAction( } } + // zipper can only consume file list options from param file not other options, + // so write file list in the param file first. + Artifact paramFile = + ruleContext.getDerivedArtifact( + ParameterFile.derivePath(zipFile.getRootRelativePath()), zipFile.getRoot()); + + ruleContext.registerAction( + new ParameterFileWriteAction( + ruleContext.getActionOwner(), + paramFile, + argv.build(), + ParameterFile.ParameterFileType.UNQUOTED, + ISO_8859_1)); + ruleContext.registerAction( new SpawnAction.Builder() + .addInput(paramFile) .addTransitiveInputs(inputsBuilder.build()) .addOutput(zipFile) .setExecutable(zipper) .useDefaultShellEnvironment() - .addCommandLine(CustomCommandLine.builder().add("cC").addExecPath(zipFile).build()) - // zipper can only consume file list options from param file not other options, - // so write file list in the param file. .addCommandLine( - argv.build(), - ParamFileInfo.builder(ParameterFile.ParameterFileType.UNQUOTED) - .setCharset(ISO_8859_1) - .setUseAlways(true) + CustomCommandLine.builder() + .add("cC") + .addExecPath(zipFile) + .addPrefixedExecPath("@", paramFile) .build()) .setMnemonic("PythonZipper") .build(ruleContext)); diff --git a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/ShBinary.java b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/ShBinary.java index 72545267cf0..94c4636dc29 100644 --- a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/ShBinary.java +++ b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/ShBinary.java @@ -27,9 +27,9 @@ import com.google.devtools.build.lib.analysis.actions.ExecutableSymlinkAction; import com.google.devtools.build.lib.analysis.actions.LauncherFileWriteAction; import com.google.devtools.build.lib.analysis.actions.LauncherFileWriteAction.LaunchInfo; -import com.google.devtools.build.lib.analysis.actions.Substitution; -import com.google.devtools.build.lib.analysis.actions.Template; import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction; +import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction.Substitution; +import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction.Template; import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget.Mode; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; diff --git a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/exec/TestStrategy.java b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/exec/TestStrategy.java index fcd21ea316f..ae655519275 100644 --- a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/exec/TestStrategy.java +++ b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/exec/TestStrategy.java @@ -409,7 +409,13 @@ private static void updateLocalRunfilesDirectory( actionExecutionContext.getInputPath(execSettings.getInputManifest()), runfilesDir, false) - .createSymlinks(actionExecutionContext, binTools, shellEnvironment, enableRunfiles); + .createSymlinks( + testAction, + actionExecutionContext, + binTools, + shellEnvironment, + execSettings.getInputManifest(), + enableRunfiles); actionExecutionContext.getEventHandler() .handle(Event.progress(testAction.getProgressMessage())); diff --git a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java index 6faaa1958a6..30b57a00f2c 100644 --- a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java +++ b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java @@ -1207,6 +1207,44 @@ CompilationSupport registerLinkActions( return this; } + /** + * Registers any actions necessary to link this rule and its dependencies. Automatically infers + * the toolchain from the configuration of this CompilationSupport - if a different toolchain is + * required, use the custom toolchain override. + * + *

Dsym bundle is generated if {@link ObjcConfiguration#generateDsym()} is set. + * + *

When Bazel flags {@code --compilation_mode=opt} and {@code --objc_enable_binary_stripping} + * are specified, additional optimizations will be performed on the linked binary: all-symbol + * stripping (using {@code /usr/bin/strip}) and dead-code stripping (using linker flags: {@code + * -dead_strip} and {@code -no_dead_strip_inits_and_terms}). + * + * @param objcProvider common information about this rule's attributes and its dependencies + * @param j2ObjcMappingFileProvider contains mapping files for j2objc transpilation + * @param j2ObjcEntryClassProvider contains j2objc entry class information for dead code removal + * @param extraLinkArgs any additional arguments to pass to the linker + * @param extraLinkInputs any additional input artifacts to pass to the link action + * @param dsymOutputType the file type of the dSYM bundle to be generated + * @return this compilation support + */ + CompilationSupport registerLinkActions( + ObjcProvider objcProvider, + J2ObjcMappingFileProvider j2ObjcMappingFileProvider, + J2ObjcEntryClassProvider j2ObjcEntryClassProvider, + ExtraLinkArgs extraLinkArgs, + Iterable extraLinkInputs, + DsymOutputType dsymOutputType) + throws InterruptedException { + return registerLinkActions( + objcProvider, + j2ObjcMappingFileProvider, + j2ObjcEntryClassProvider, + extraLinkArgs, + extraLinkInputs, + dsymOutputType, + CppHelper.getToolchainUsingDefaultCcToolchainAttribute(ruleContext)); + } + /** * Returns the copts for the compile action in the current rule context (using a combination of * the rule's "copts" attribute as well as the current configuration copts). diff --git a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java index 4cf8d7dfb92..5670569d3d1 100644 --- a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java +++ b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java @@ -37,6 +37,7 @@ import com.google.devtools.build.lib.buildtool.BuildResult; import com.google.devtools.build.lib.buildtool.BuildTool; import com.google.devtools.build.lib.buildtool.OutputDirectoryLinksUtils; +import com.google.devtools.build.lib.buildtool.TargetValidator; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.Reporter; import com.google.devtools.build.lib.exec.ExecutionOptions; @@ -57,14 +58,19 @@ import com.google.devtools.build.lib.runtime.BlazeServerStartupOptions; import com.google.devtools.build.lib.runtime.Command; import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.runtime.ProcessWrapperUtil; import com.google.devtools.build.lib.server.CommandProtos.EnvironmentVariable; import com.google.devtools.build.lib.server.CommandProtos.ExecRequest; +import com.google.devtools.build.lib.shell.AbnormalTerminationException; +import com.google.devtools.build.lib.shell.BadExitStatusException; import com.google.devtools.build.lib.shell.CommandException; import com.google.devtools.build.lib.syntax.Type; +import com.google.devtools.build.lib.util.CommandBuilder; import com.google.devtools.build.lib.util.CommandDescriptionForm; import com.google.devtools.build.lib.util.CommandFailureUtils; import com.google.devtools.build.lib.util.ExitCode; import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.OS; import com.google.devtools.build.lib.util.OptionsUtils; import com.google.devtools.build.lib.util.ShellEscaper; import com.google.devtools.build.lib.util.io.OutErr; @@ -105,6 +111,27 @@ public class RunCommand implements BlazeCommand { /** Options for the "run" command. */ public static class RunOptions extends OptionsBase { + @Option( + name = "as_test", + defaultValue = "true", + documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS, + effectTags = {OptionEffectTag.EXECUTION}, + help = "If set, the 'run' command will execute tests in an approximation of the official " + + "test environment. Otherwise, tests will be run as regular binaries.") + public boolean asTest; + + @Option( + name = "direct_run", + defaultValue = "true", + documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS, + effectTags = {OptionEffectTag.EXECUTION}, + help = "If set, the 'run' command will execute the binary to be executed in the terminal " + + "where the command was called. Otherwise, it'll be executed as a child of the server " + + "process. If set, the binary will have access to direct terminal I/O and the command " + + "lock will not be held during its execution. This makes it possible to run other " + + "commands in parallel.") + public boolean direct; + @Option( name = "script_path", defaultValue = "null", @@ -147,9 +174,13 @@ public RunCommand(TestPolicy testPolicy) { @VisibleForTesting // productionVisibility = Visibility.PRIVATE protected BuildResult processRequest(final CommandEnvironment env, BuildRequest request) { - return new BuildTool(env).processRequest(request, - (Collection targets, boolean keepGoing) -> - RunCommand.this.validateTargets(env.getReporter(), targets, keepGoing)); + return new BuildTool(env).processRequest(request, new TargetValidator() { + @Override + public void validateTargets(Collection targets, boolean keepGoing) + throws LoadingFailedException { + RunCommand.this.validateTargets(env.getReporter(), targets, keepGoing); + } + }); } @Override @@ -179,6 +210,83 @@ private List computeArgs(CommandEnvironment env, ConfiguredTarget target return args; } + private BlazeCommandResult runTargetUnderServer(CommandEnvironment env, + ConfiguredTarget targetToRun, PathFragment shellExecutable, + ConfiguredTarget runUnderTarget, Path workingDir, List commandLineArgs) { + RunOptions runOptions = env.getOptions().getOptions(RunOptions.class); + List args = computeArgs(env, targetToRun, commandLineArgs); + if (args == null) { + return BlazeCommandResult.exitCode(ExitCode.ANALYSIS_FAILURE); + } + + // We now have a unique executable ready to be run. + // + // We build up two different versions of the command to run: one with an absolute path, which + // we'll actually run, and a prettier one with the long absolute path to the executable + // replaced with a shorter relative path that uses the symlinks in the workspace. + List cmdLine = new ArrayList<>(); + // process-wrapper does not work on Windows (nor is it necessary), so don't use it + // on that platform. Also we skip it when writing the command-line to a file instead + // of executing it directly. + if (OS.getCurrent() != OS.WINDOWS && runOptions.scriptPath == null) { + Preconditions.checkState( + ProcessWrapperUtil.isSupported(env), "process-wrapper not found in embedded tools"); + cmdLine.add(ProcessWrapperUtil.getProcessWrapper(env).getPathString()); + } + List prettyCmdLine = new ArrayList<>(); + constructCommandLine(cmdLine, prettyCmdLine, env, shellExecutable, + targetToRun, runUnderTarget, args); + + // Add a newline between the blaze output and the binary's output. + env.getReporter().getOutErr().printErrLn(""); + + if (runOptions.scriptPath != null) { + String unisolatedCommand = CommandFailureUtils.describeCommand( + CommandDescriptionForm.COMPLETE_UNISOLATED, + cmdLine, null, workingDir.getPathString()); + if (writeScript(env, shellExecutable, runOptions.scriptPath, unisolatedCommand)) { + return BlazeCommandResult.exitCode(ExitCode.SUCCESS); + } else { + return BlazeCommandResult.exitCode(ExitCode.RUN_FAILURE); + } + } + + env.getReporter().handle(Event.info( + null, "Running command line: " + ShellEscaper.escapeJoinAll(prettyCmdLine))); + + try { + com.google.devtools.build.lib.shell.Command command = new CommandBuilder() + .addArgs(cmdLine).setEnv(env.getClientEnv()).setWorkingDir(workingDir).build(); + + // Restore a raw EventHandler if it is registered. This allows for blaze run to produce the + // actual output of the command being run even if --color=no is specified. + env.getReporter().switchToAnsiAllowingHandler(); + + // The command API is a little strange in that the following statement will return normally + // only if the program exits with exit code 0. If it ends with any other code, we have to + // catch BadExitStatusException. + command + .execute( + env.getReporter().getOutErr().getOutputStream(), + env.getReporter().getOutErr().getErrorStream()) + .getTerminationStatus() + .getExitCode(); + return BlazeCommandResult.exitCode(ExitCode.SUCCESS); + } catch (BadExitStatusException e) { + String message = "Non-zero return code '" + + e.getResult().getTerminationStatus().getExitCode() + + "' from command: " + e.getMessage(); + env.getReporter().handle(Event.error(message)); + return BlazeCommandResult.exitCode(ExitCode.RUN_FAILURE); + } catch (AbnormalTerminationException e) { + // The process was likely terminated by a signal in this case. + return BlazeCommandResult.exitCode(ExitCode.INTERRUPTED); + } catch (CommandException e) { + env.getReporter().handle(Event.error("Error running program: " + e.getMessage())); + return BlazeCommandResult.exitCode(ExitCode.RUN_FAILURE); + } + } + private void constructCommandLine(List cmdLine, List prettyCmdLine, CommandEnvironment env, PathFragment shellExecutable, ConfiguredTarget targetToRun, ConfiguredTarget runUnderTarget, List args) { @@ -354,6 +462,11 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsProvider options) return BlazeCommandResult.exitCode(ExitCode.COMMAND_LINE_ERROR); } + if (!runOptions.direct) { + return runTargetUnderServer( + env, targetToRun, shExecutable, runUnderTarget, runfilesDir, commandLineArgs); + } + Map runEnvironment = new TreeMap<>(); List cmdLine = new ArrayList<>(); List prettyCmdLine = new ArrayList<>(); @@ -362,7 +475,7 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsProvider options) runEnvironment.put("BUILD_WORKSPACE_DIRECTORY", env.getWorkspace().getPathString()); runEnvironment.put("BUILD_WORKING_DIRECTORY", env.getWorkingDirectory().getPathString()); - if (targetToRun.getProvider(TestProvider.class) != null) { + if (targetToRun.getProvider(TestProvider.class) != null && runOptions.asTest) { // This is a test. Provide it with a reasonable approximation of the actual test environment ImmutableList statusArtifacts = TestProvider.getTestStatusArtifacts(targetToRun); if (statusArtifacts.size() != 1) { @@ -474,6 +587,10 @@ private boolean prepareTestEnvironment(CommandEnvironment env, TestRunnerAction /** * Ensures that runfiles are built for the specified target. If they already * are, does nothing, otherwise builds them. + * + * @param target the target to build runfiles for. + * @return the path of the runfiles directory. + * @throws CommandException */ private Path ensureRunfilesBuilt(CommandEnvironment env, RunfilesSupport runfilesSupport, BuildConfiguration configuration) throws CommandException { @@ -595,7 +712,7 @@ private static String validateTarget(Target target) { */ private ExitCode fullyValidateTarget(CommandEnvironment env, ConfiguredTarget configuredTarget) { - Target target; + Target target = null; try { target = env.getPackageManager().getTarget(env.getReporter(), configuredTarget.getLabel()); } catch (NoSuchTargetException | NoSuchPackageException | InterruptedException e) {