diff --git a/CHANGELOG.md b/CHANGELOG.md index 63bce1e77..c59517adc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog ## Next release + +### Breaking Changes +- Eth2 Azure command line option --azure-secrets-tags is now deprecated and is replaced with --azure-tags. The --azure-secrets-tags option will be removed in a future release. + ### Features Added - Azure bulk mode support for loading multiline (`\n` delimited, up to 200) keys per secret. - Hashicorp connection properties can now override http protocol to HTTP/1.1 from the default of HTTP/2. [#817](https://github.com/ConsenSys/web3signer/pull/817) @@ -8,6 +12,7 @@ - Add eth_signTransaction RPC method under the eth1 subcommand [#822](https://github.com/ConsenSys/web3signer/pull/822) - Add eth_sendTransaction RPC method under the eth1 subcommand [#835](https://github.com/Consensys/web3signer/pull/835) - Add EIP-1559 support for eth1 public transactions for eth_sendTransaction and eth_signTransaction [#836](https://github.com/Consensys/web3signer/pull/836) +- Add Azure bulk loading for secp256k1 keys in eth1 mode [#850](https://github.com/Consensys/web3signer/pull/850) ### Bugs fixed - Support long name aliases in environment variables and YAML configuration [#825](https://github.com/Consensys/web3signer/pull/825) diff --git a/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsConfigFileImpl.java b/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsConfigFileImpl.java index 8a0181e18..a00ffc94d 100644 --- a/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsConfigFileImpl.java +++ b/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsConfigFileImpl.java @@ -115,33 +115,6 @@ public List createCmdLineParams() { if (signerConfig.getMode().equals("eth2")) { yamlConfig.append(createEth2SlashingProtectionArgs()); - if (signerConfig.getAzureKeyVaultParameters().isPresent()) { - final AzureKeyVaultParameters azureParams = signerConfig.getAzureKeyVaultParameters().get(); - yamlConfig.append( - String.format(YAML_BOOLEAN_FMT, "eth2.azure-vault-enabled", Boolean.TRUE)); - yamlConfig.append( - String.format( - YAML_STRING_FMT, - "eth2.azure-vault-auth-mode", - azureParams.getAuthenticationMode().name())); - yamlConfig.append( - String.format(YAML_STRING_FMT, "eth2.azure-vault-name", azureParams.getKeyVaultName())); - yamlConfig.append( - String.format(YAML_STRING_FMT, "eth2.azure-client-id", azureParams.getClientId())); - yamlConfig.append( - String.format( - YAML_STRING_FMT, "eth2.azure-client-secret", azureParams.getClientSecret())); - yamlConfig.append( - String.format(YAML_STRING_FMT, "eth2.azure-tenant-id", azureParams.getTenantId())); - - azureParams - .getTags() - .forEach( - (tagName, tagValue) -> - yamlConfig.append( - String.format( - YAML_STRING_FMT, "eth2.azure-secrets-tags", tagName + "=" + tagValue))); - } if (signerConfig.getKeystoresParameters().isPresent()) { final KeystoresParameters keystoresParameters = signerConfig.getKeystoresParameters().get(); yamlConfig.append( @@ -181,6 +154,12 @@ public List createCmdLineParams() { yamlConfig.append(createDownstreamTlsArgs()); } + signerConfig + .getAzureKeyVaultParameters() + .ifPresent( + azureParams -> + yamlConfig.append(azureBulkLoadingOptions(signerConfig.getMode(), azureParams))); + // create temporary config file try { final Path configFile = Files.createTempFile("web3signer_config", ".yaml"); @@ -196,6 +175,35 @@ public List createCmdLineParams() { return params; } + private String azureBulkLoadingOptions( + final String mode, final AzureKeyVaultParameters azureParams) { + final StringBuilder yamlConfig = new StringBuilder(); + yamlConfig.append(String.format(YAML_BOOLEAN_FMT, mode + ".azure-vault-enabled", Boolean.TRUE)); + yamlConfig.append( + String.format( + YAML_STRING_FMT, + mode + ".azure-vault-auth-mode", + azureParams.getAuthenticationMode().name())); + yamlConfig.append( + String.format(YAML_STRING_FMT, mode + ".azure-vault-name", azureParams.getKeyVaultName())); + yamlConfig.append( + String.format(YAML_STRING_FMT, mode + ".azure-client-id", azureParams.getClientId())); + yamlConfig.append( + String.format( + YAML_STRING_FMT, mode + ".azure-client-secret", azureParams.getClientSecret())); + yamlConfig.append( + String.format(YAML_STRING_FMT, mode + ".azure-tenant-id", azureParams.getTenantId())); + + azureParams + .getTags() + .forEach( + (tagName, tagValue) -> + yamlConfig.append( + String.format( + YAML_STRING_FMT, mode + ".azure-tags", tagName + "=" + tagValue))); + return yamlConfig.toString(); + } + private CommandArgs createSubCommandArgs() { final List params = new ArrayList<>(); final StringBuilder yamlConfig = new StringBuilder(); diff --git a/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsDefaultImpl.java b/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsDefaultImpl.java index f07a2d2e0..90c9e29ea 100644 --- a/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsDefaultImpl.java +++ b/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsDefaultImpl.java @@ -99,26 +99,7 @@ public List createCmdLineParams() { params.addAll(createEth2Args()); if (signerConfig.getAzureKeyVaultParameters().isPresent()) { - final AzureKeyVaultParameters azureParams = signerConfig.getAzureKeyVaultParameters().get(); - params.add("--azure-vault-enabled=true"); - params.add("--azure-vault-auth-mode"); - params.add(azureParams.getAuthenticationMode().name()); - params.add("--azure-vault-name"); - params.add(azureParams.getKeyVaultName()); - params.add("--azure-client-id"); - params.add(azureParams.getClientId()); - params.add("--azure-client-secret"); - params.add(azureParams.getClientSecret()); - params.add("--azure-tenant-id"); - params.add(azureParams.getTenantId()); - - azureParams - .getTags() - .forEach( - (tagName, tagValue) -> { - params.add("--azure-secrets-tags"); - params.add(tagName + "=" + tagValue); - }); + createAzureArgs(params); } if (signerConfig.getKeystoresParameters().isPresent()) { final KeystoresParameters keystoresParameters = signerConfig.getKeystoresParameters().get(); @@ -143,6 +124,10 @@ public List createCmdLineParams() { params.add("--chain-id"); params.add(Long.toString(signerConfig.getChainIdProvider().id())); params.addAll(createDownstreamTlsArgs()); + + if (signerConfig.getAzureKeyVaultParameters().isPresent()) { + createAzureArgs(params); + } } return params; @@ -331,6 +316,29 @@ private Collection awsBulkLoadingOptions( return params; } + private void createAzureArgs(final List params) { + final AzureKeyVaultParameters azureParams = signerConfig.getAzureKeyVaultParameters().get(); + params.add("--azure-vault-enabled=true"); + params.add("--azure-vault-auth-mode"); + params.add(azureParams.getAuthenticationMode().name()); + params.add("--azure-vault-name"); + params.add(azureParams.getKeyVaultName()); + params.add("--azure-client-id"); + params.add(azureParams.getClientId()); + params.add("--azure-client-secret"); + params.add(azureParams.getClientSecret()); + params.add("--azure-tenant-id"); + params.add(azureParams.getTenantId()); + + azureParams + .getTags() + .forEach( + (tagName, tagValue) -> { + params.add("--azure-tags"); + params.add(tagName + "=" + tagValue); + }); + } + private List createSubCommandArgs() { final List params = new ArrayList<>(); diff --git a/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AzureKeyVaultAcceptanceTest.java b/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AzureKeyVaultAcceptanceTest.java index 3c8a24575..3c7dfa4da 100644 --- a/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AzureKeyVaultAcceptanceTest.java +++ b/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AzureKeyVaultAcceptanceTest.java @@ -14,26 +14,27 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.hasSize; import tech.pegasys.web3signer.dsl.signer.SignerConfigurationBuilder; -import tech.pegasys.web3signer.dsl.utils.DefaultAzureKeyVaultParameters; import tech.pegasys.web3signer.signing.KeyType; import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters; +import tech.pegasys.web3signer.signing.config.DefaultAzureKeyVaultParameters; import tech.pegasys.web3signer.tests.AcceptanceTestBase; import java.util.Map; +import java.util.stream.Stream; import io.restassured.http.ContentType; import io.restassured.response.Response; import io.vertx.core.json.JsonObject; import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; public class AzureKeyVaultAcceptanceTest extends AcceptanceTestBase { @@ -41,10 +42,14 @@ public class AzureKeyVaultAcceptanceTest extends AcceptanceTestBase { private static final String CLIENT_SECRET = System.getenv("AZURE_CLIENT_SECRET"); private static final String TENANT_ID = System.getenv("AZURE_TENANT_ID"); private static final String VAULT_NAME = System.getenv("AZURE_KEY_VAULT_NAME"); - private static final String EXPECTED_KEY = + private static final String BLS_KEY = "0x989d34725a2bfc3f15105f3f5fc8741f436c25ee1ee4f948e425d6bcb8c56bce6e06c269635b7e985a7ffa639e2409bf"; - private static final String EXPECTED_TAGGED_KEY = + private static final String BLS_TAGGED_KEY = "0xb3b6fb8dab2a4c9d00247c18c4b7e91c62da3f7ad31c822c00097f93ac8ff2c4a526611f7d0a9c85946e93f371852c69"; + private static final String SECP_KEY = + "0xa95663509e608da3c2af5a48eb4315321f8430cbed5518a44590cc9d367f01dc72ebbc583fc7d94f9fdc20eb6e162c9f8cb35be8a91a3b1d32a63ecc10be4e08"; + private static final String SECP_TAGGED_KEY = + "0x234053dbe014ebe573e5e8f6eab5e0417bf705466009f7c15b8d23593abd1bda426593d92b32efb240afe6efa46d5679fad0dc427e0aa0fc61c2464ce93c7c5e"; @BeforeAll public static void setup() { @@ -54,22 +59,25 @@ public static void setup() { Assumptions.assumeTrue(VAULT_NAME != null, "Set AZURE_KEY_VAULT_NAME environment variable"); } - @Test - void ensureSecretsInKeyVaultAreLoadedAndReportedViaPublicKeysApi() { + @ParameterizedTest + @EnumSource(KeyType.class) + void ensureSecretsInKeyVaultAreLoadedAndReportedViaPublicKeysApi(final KeyType keyType) { final AzureKeyVaultParameters azureParams = new DefaultAzureKeyVaultParameters(VAULT_NAME, CLIENT_ID, TENANT_ID, CLIENT_SECRET); final SignerConfigurationBuilder configBuilder = - new SignerConfigurationBuilder().withMode("eth2").withAzureKeyVaultParameters(azureParams); + new SignerConfigurationBuilder() + .withMode(calculateMode(keyType)) + .withAzureKeyVaultParameters(azureParams); startSigner(configBuilder.build()); - final Response response = signer.callApiPublicKeys(KeyType.BLS); + final Response response = signer.callApiPublicKeys(keyType); response .then() .statusCode(200) .contentType(ContentType.JSON) - .body("", hasItems(EXPECTED_KEY, EXPECTED_TAGGED_KEY)); + .body("", hasItems(expectedKey(keyType, false))); final Response healthcheckResponse = signer.healthcheck(); healthcheckResponse @@ -78,35 +86,37 @@ void ensureSecretsInKeyVaultAreLoadedAndReportedViaPublicKeysApi() { .contentType(ContentType.JSON) .body("status", equalTo("UP")); + // BLS keys include additional multi-line key with 200 keys + final int expectedKeyLoaded = keyType == KeyType.BLS ? 202 : 2; + final String jsonBody = healthcheckResponse.body().asString(); - int keysLoaded = getAzureBulkLoadingData(jsonBody, "keys-loaded"); - assertThat(keysLoaded) - .isEqualTo(202); // ACCTEST-MULTILINE-KEY (200) + TEST-KEY (1) + TEST-KEY-2 (1) + final int keysLoaded = getAzureBulkLoadingData(jsonBody, "keys-loaded"); + assertThat(keysLoaded).isEqualTo(expectedKeyLoaded); } - @ParameterizedTest(name = "{index} - Using config file: {0}") - @ValueSource(booleans = {true, false}) - void azureSecretsViaTag(boolean useConfigFile) { + @ParameterizedTest(name = "{index} - KeyType: {0}, using config file: {1}") + @MethodSource("azureSecretsViaTag") + void azureSecretsViaTag(final KeyType keyType, boolean useConfigFile) { final AzureKeyVaultParameters azureParams = new DefaultAzureKeyVaultParameters( VAULT_NAME, CLIENT_ID, TENANT_ID, CLIENT_SECRET, Map.of("ENV", "TEST")); final SignerConfigurationBuilder configBuilder = new SignerConfigurationBuilder() - .withMode("eth2") + .withMode(calculateMode(keyType)) .withUseConfigFile(useConfigFile) .withAzureKeyVaultParameters(azureParams); startSigner(configBuilder.build()); - final Response response = signer.callApiPublicKeys(KeyType.BLS); + final Response response = signer.callApiPublicKeys(keyType); response .then() .statusCode(200) .contentType(ContentType.JSON) - .body("", hasItem(EXPECTED_TAGGED_KEY)); + .body("", hasItems(expectedKey(keyType, true))); - // the tag filter will return only valid keys. The healtcheck should be UP + // the tag filter will return only valid keys. The healthcheck should be UP final Response healthcheckResponse = signer.healthcheck(); healthcheckResponse .then() @@ -116,36 +126,45 @@ void azureSecretsViaTag(boolean useConfigFile) { // keys loaded should be 1 as well. final String jsonBody = healthcheckResponse.body().asString(); - int keysLoaded = getAzureBulkLoadingData(jsonBody, "keys-loaded"); - int errorCount = getAzureBulkLoadingData(jsonBody, "error-count"); + final int keysLoaded = getAzureBulkLoadingData(jsonBody, "keys-loaded"); + final int errorCount = getAzureBulkLoadingData(jsonBody, "error-count"); assertThat(keysLoaded).isOne(); assertThat(errorCount).isZero(); } + private static Stream azureSecretsViaTag() { + return Stream.of( + Arguments.arguments(KeyType.BLS, false), + Arguments.arguments(KeyType.BLS, true), + Arguments.arguments(KeyType.SECP256K1, false), + Arguments.arguments(KeyType.SECP256K1, true)); + } + private static int getAzureBulkLoadingData(String healthCheckJsonBody, String dataKey) { - JsonObject jsonObject = new JsonObject(healthCheckJsonBody); - int keysLoaded = - jsonObject.getJsonArray("checks").stream() - .filter(o -> "keys-check".equals(((JsonObject) o).getString("id"))) - .flatMap(o -> ((JsonObject) o).getJsonArray("checks").stream()) - .filter(o -> "azure-bulk-loading".equals(((JsonObject) o).getString("id"))) - .mapToInt(o -> ((JsonObject) ((JsonObject) o).getValue("data")).getInteger(dataKey)) - .findFirst() - .orElse(-1); - return keysLoaded; + final JsonObject jsonObject = new JsonObject(healthCheckJsonBody); + return jsonObject.getJsonArray("checks").stream() + .filter(o -> "keys-check".equals(((JsonObject) o).getString("id"))) + .flatMap(o -> ((JsonObject) o).getJsonArray("checks").stream()) + .filter(o -> "azure-bulk-loading".equals(((JsonObject) o).getString("id"))) + .mapToInt(o -> ((JsonObject) ((JsonObject) o).getValue("data")).getInteger(dataKey)) + .findFirst() + .orElse(-1); } - @Test - void invalidVaultParametersFailsToLoadKeys() { + @ParameterizedTest + @EnumSource(KeyType.class) + void invalidVaultParametersFailsToLoadKeys(final KeyType keyType) { final AzureKeyVaultParameters azureParams = new DefaultAzureKeyVaultParameters("nonExistentVault", CLIENT_ID, TENANT_ID, CLIENT_SECRET); final SignerConfigurationBuilder configBuilder = - new SignerConfigurationBuilder().withMode("eth2").withAzureKeyVaultParameters(azureParams); + new SignerConfigurationBuilder() + .withMode(calculateMode(keyType)) + .withAzureKeyVaultParameters(azureParams); startSigner(configBuilder.build()); - final Response response = signer.callApiPublicKeys(KeyType.BLS); + final Response response = signer.callApiPublicKeys(keyType); response.then().statusCode(200).contentType(ContentType.JSON).body("", hasSize(0)); signer @@ -156,23 +175,35 @@ void invalidVaultParametersFailsToLoadKeys() { .body("status", equalTo("DOWN")); } - @Test - void envVarsAreUsedToDefaultAzureParams() { + @ParameterizedTest + @EnumSource(KeyType.class) + void envVarsAreUsedToDefaultAzureParams(final KeyType keyType) { // This ensures env vars correspond to the WEB3SIGNER__