Implement Healthcheck status for keys loading (#738)

CHANGELOG.md

@@ -3,6 +3,8 @@
 ## Upcoming version
 ### Features Added
 - Add support for Capella milestone in Mainnet
+- Enhanced Healthcheck endpoint reporting status of loading of signers keys [#738](https://github.com/ConsenSys/web3signer/pull/738)
+- Optional AWS endpoint overriding for bulk loading `--aws-endpoint-override`. Useful for local testing against localstack. [#730](https://github.com/ConsenSys/web3signer/issues/730)
 
 ### Bugs Fixed
 - Update of Azure libraries (transitive via signers library) and manual override to fix CVE-2023-1370

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/Signer.java

@@ -61,6 +61,7 @@ public class Signer extends FilecoinJsonRpcEndpoint {
   public static final ObjectMapper ETH_2_INTERFACE_OBJECT_MAPPER =
       SigningObjectMapperFactory.createObjectMapper().setSerializationInclusion(Include.NON_NULL);
   private static final String METRICS_ENDPOINT = "/metrics";
+  private static final String HEALTHCHECK_ENDPOINT = "/healthcheck";
 
   private final SignerConfiguration signerConfig;
   private final Web3SignerRunner runner;
@@ -185,4 +186,8 @@ public String getSlashingDbUrl() {
   public Response callReload() {
     return given().baseUri(getUrl()).post(RELOAD_ENDPOINT);
   }
+
+  public Response healthcheck() {
+    return given().baseUri(getUrl()).get(HEALTHCHECK_ENDPOINT);
+  }
 }

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsConfigFileImpl.java

@@ -12,6 +12,7 @@
  */
 package tech.pegasys.web3signer.dsl.signer.runner;
 
+import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_ENDPOINT_OVERRIDE_OPTION;
 import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_ACCESS_KEY_ID_OPTION;
 import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_AUTH_MODE_OPTION;
 import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_ENABLED_OPTION;
@@ -422,6 +423,16 @@ private String awsBulkLoadingOptions(
               String.join(",", awsSecretsManagerParameters.getTagValuesFilter())));
     }
 
+    awsSecretsManagerParameters
+        .getEndpointOverride()
+        .ifPresent(
+            uri ->
+                yamlConfig.append(
+                    String.format(
+                        YAML_STRING_FMT,
+                        "eth2." + AWS_ENDPOINT_OVERRIDE_OPTION.substring(2),
+                        uri)));
+
     return yamlConfig.toString();
   }
 

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsDefaultImpl.java

@@ -12,6 +12,7 @@
  */
 package tech.pegasys.web3signer.dsl.signer.runner;
 
+import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_ENDPOINT_OVERRIDE_OPTION;
 import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_ACCESS_KEY_ID_OPTION;
 import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_AUTH_MODE_OPTION;
 import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_ENABLED_OPTION;
@@ -255,6 +256,14 @@ private Collection<String> awsBulkLoadingOptions(
       params.add(awsSecretsManagerParameters.getRegion());
     }
 
+    awsSecretsManagerParameters
+        .getEndpointOverride()
+        .ifPresent(
+            uri -> {
+              params.add(AWS_ENDPOINT_OVERRIDE_OPTION);
+              params.add(uri.toString());
+            });
+
     if (!awsSecretsManagerParameters.getPrefixesFilter().isEmpty()) {
       params.add(AWS_SECRETS_PREFIXES_FILTER_OPTION);
       params.add(String.join(",", awsSecretsManagerParameters.getPrefixesFilter()));

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AwsSecretsManagerAcceptanceTest.java

@@ -12,6 +12,7 @@
  */
 package tech.pegasys.web3signer.tests.bulkloading;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.hasSize;
 
@@ -24,17 +25,20 @@
 import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParametersBuilder;
 import tech.pegasys.web3signer.tests.AcceptanceTestBase;
 
+import java.net.URI;
 import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 
 import io.restassured.http.ContentType;
+import io.vertx.core.json.JsonObject;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInstance;
 import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -69,13 +73,20 @@ public class AwsSecretsManagerAcceptanceTest extends AcceptanceTestBase {
   private static final String RO_AWS_SECRET_ACCESS_KEY = System.getenv("AWS_SECRET_ACCESS_KEY");
   private static final String AWS_REGION =
       Optional.ofNullable(System.getenv("AWS_REGION")).orElse("us-east-2");
+
+  // can be pointed to localstack
+  private final Optional<URI> awsEndpointOverride =
+      System.getenv("AWS_ENDPOINT_OVERRIDE") != null
+          ? Optional.of(URI.create(System.getenv("AWS_ENDPOINT_OVERRIDE")))
+          : Optional.empty();
   private AwsSecretsManagerUtil awsSecretsManagerUtil;
   private final List<BLSKeyPair> blsKeyPairs = new ArrayList<>();
 
   @BeforeAll
   void setupAwsResources() {
     awsSecretsManagerUtil =
-        new AwsSecretsManagerUtil(AWS_REGION, RW_AWS_ACCESS_KEY_ID, RW_AWS_SECRET_ACCESS_KEY);
+        new AwsSecretsManagerUtil(
+            AWS_REGION, RW_AWS_ACCESS_KEY_ID, RW_AWS_SECRET_ACCESS_KEY, awsEndpointOverride);
     final SecureRandom secureRandom = new SecureRandom();
 
     for (int i = 0; i < 4; i++) {
@@ -88,7 +99,7 @@ void setupAwsResources() {
     }
   }
 
-  @ParameterizedTest
+  @ParameterizedTest(name = "{index} - Using config file: {0}")
   @ValueSource(booleans = {true, false})
   void secretsAreLoadedFromAWSSecretsManagerAndReportedByPublicApi(final boolean useConfigFile) {
     final AwsSecretsManagerParameters awsSecretsManagerParameters =
@@ -100,6 +111,7 @@ void secretsAreLoadedFromAWSSecretsManagerAndReportedByPublicApi(final boolean u
             .withPrefixesFilter(List.of(awsSecretsManagerUtil.getSecretsManagerPrefix()))
             .withTagNamesFilter(List.of("TagName0", "TagName1"))
             .withTagValuesFilter(List.of("TagValue0", "TagValue1", "TagValue2"))
+            .withEndpointOverride(awsEndpointOverride)
             .build();
 
     final SignerConfigurationBuilder configBuilder =
@@ -110,6 +122,11 @@ void secretsAreLoadedFromAWSSecretsManagerAndReportedByPublicApi(final boolean u
 
     startSigner(configBuilder.build());
 
+    final String healthCheckJsonBody = signer.healthcheck().body().asString();
+    int keysLoaded = getAwsBulkLoadingData(healthCheckJsonBody, "keys-loaded");
+
+    assertThat(keysLoaded).isEqualTo(2);
+
     signer
         .callApiPublicKeys(KeyType.BLS)
         .then()
@@ -124,7 +141,51 @@ void secretsAreLoadedFromAWSSecretsManagerAndReportedByPublicApi(final boolean u
             hasSize(2));
   }
 
-  @ParameterizedTest
+  @Test
+  void healthCheckErrorCountWhenInvalidCredentialsAreUsed() {
+    final boolean useConfigFile = false;
+    final AwsSecretsManagerParameters invalidCredsParams =
+        AwsSecretsManagerParametersBuilder.anAwsSecretsManagerParameters()
+            .withAuthenticationMode(AwsAuthenticationMode.SPECIFIED)
+            .withRegion("us-east-2")
+            .withAccessKeyId("invalid")
+            .withSecretAccessKey("invalid")
+            .withPrefixesFilter(List.of("shouldNotExist/"))
+            .withEndpointOverride(Optional.empty())
+            .build();
+
+    final SignerConfigurationBuilder configBuilder =
+        new SignerConfigurationBuilder()
+            .withUseConfigFile(useConfigFile)
+            .withMode("eth2")
+            .withAwsSecretsManagerParameters(invalidCredsParams);
+
+    startSigner(configBuilder.build());
+
+    final String healthCheckJsonBody = signer.healthcheck().body().asString();
+
+    int keysLoaded = getAwsBulkLoadingData(healthCheckJsonBody, "keys-loaded");
+    int errorCount = getAwsBulkLoadingData(healthCheckJsonBody, "error-count");
+
+    assertThat(keysLoaded).isEqualTo(0);
+    assertThat(errorCount).isEqualTo(1);
+    assertThat(new JsonObject(healthCheckJsonBody).getString("status")).isEqualTo("DOWN");
+  }
+
+  private static int getAwsBulkLoadingData(String healthCheckJsonBody, String dataKey) {
+    JsonObject jsonObject = new JsonObject(healthCheckJsonBody);
+    int keysLoaded =
+        jsonObject.getJsonArray("checks").stream()
+            .filter(o -> "keys-check".equals(((JsonObject) o).getString("id")))
+            .flatMap(o -> ((JsonObject) o).getJsonArray("checks").stream())
+            .filter(o -> "aws-bulk-loading".equals(((JsonObject) o).getString("id")))
+            .mapToInt(o -> ((JsonObject) ((JsonObject) o).getValue("data")).getInteger(dataKey))
+            .findFirst()
+            .orElse(-1);
+    return keysLoaded;
+  }
+
+  @ParameterizedTest(name = "{index} - Using config file: {0}")
   @ValueSource(booleans = {true, false})
   void secretsAreLoadedFromAWSSecretsManagerWithEnvironmentAuthModeAndReportedByPublicApi(
       final boolean useConfigFile) {
@@ -134,6 +195,7 @@ void secretsAreLoadedFromAWSSecretsManagerWithEnvironmentAuthModeAndReportedByPu
             .withPrefixesFilter(List.of(awsSecretsManagerUtil.getSecretsManagerPrefix()))
             .withTagNamesFilter(List.of("TagName2", "TagName3"))
             .withTagValuesFilter(List.of("TagValue0", "TagValue2", "TagValue3"))
+            .withEndpointOverride(awsEndpointOverride)
             .build();
 
     final SignerConfigurationBuilder configBuilder =

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AwsSecretsManagerMultiValueAcceptanceTest.java

@@ -25,6 +25,7 @@
 import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParametersBuilder;
 import tech.pegasys.web3signer.tests.AcceptanceTestBase;
 
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -70,6 +71,12 @@ public class AwsSecretsManagerMultiValueAcceptanceTest extends AcceptanceTestBas
   private static final String RO_AWS_SECRET_ACCESS_KEY = System.getenv("AWS_SECRET_ACCESS_KEY");
   private static final String AWS_REGION =
       Optional.ofNullable(System.getenv("AWS_REGION")).orElse("us-east-2");
+
+  // can be pointed to localstack
+  private final Optional<URI> awsEndpointOverride =
+      System.getenv("AWS_ENDPOINT_OVERRIDE") != null
+          ? Optional.of(URI.create(System.getenv("AWS_ENDPOINT_OVERRIDE")))
+          : Optional.empty();
   private AwsSecretsManagerUtil awsSecretsManagerUtil;
   private final List<BLSKeyPair> blsKeyPairList = new ArrayList<>(400);
 
@@ -80,7 +87,8 @@ void setupAwsResources() {
     }
 
     awsSecretsManagerUtil =
-        new AwsSecretsManagerUtil(AWS_REGION, RW_AWS_ACCESS_KEY_ID, RW_AWS_SECRET_ACCESS_KEY);
+        new AwsSecretsManagerUtil(
+            AWS_REGION, RW_AWS_ACCESS_KEY_ID, RW_AWS_SECRET_ACCESS_KEY, awsEndpointOverride);
 
     for (int i = 0; i < 4; i++) {
       String multilinePrivKeys =
@@ -103,6 +111,7 @@ void secretsAreLoadedFromAWSSecretsManagerAndReportedByPublicApi(final boolean u
             .withSecretAccessKey(RO_AWS_SECRET_ACCESS_KEY)
             .withPrefixesFilter(List.of(awsSecretsManagerUtil.getSecretsManagerPrefix()))
             .withTagNamesFilter(List.of("multivalue"))
+            .withEndpointOverride(awsEndpointOverride)
             .build();
 
     final SignerConfigurationBuilder configBuilder =

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AwsSecretsManagerPerformanceAcceptanceTest.java

@@ -23,6 +23,7 @@
 import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParametersBuilder;
 import tech.pegasys.web3signer.tests.AcceptanceTestBase;
 
+import java.net.URI;
 import java.security.SecureRandom;
 import java.time.Duration;
 import java.util.Collections;
@@ -79,6 +80,12 @@ public class AwsSecretsManagerPerformanceAcceptanceTest extends AcceptanceTestBa
   private static final String RO_AWS_SECRET_ACCESS_KEY = System.getenv("AWS_SECRET_ACCESS_KEY");
   private static final String AWS_REGION =
       Optional.ofNullable(System.getenv("AWS_REGION")).orElse("us-east-2");
+
+  // can be pointed to localstack
+  private final Optional<URI> awsEndpointOverride =
+      System.getenv("AWS_ENDPOINT_OVERRIDE") != null
+          ? Optional.of(URI.create(System.getenv("AWS_ENDPOINT_OVERRIDE")))
+          : Optional.empty();
   private static final Integer NUMBER_OF_KEYS =
       Integer.parseInt(Optional.ofNullable(System.getenv("AWS_PERF_AT_KEYS_NUM")).orElse("1000"));
   private static final Duration STARTUP_TIMEOUT = Duration.ofMinutes(10);
@@ -90,7 +97,8 @@ void setupAwsResources() {
     final StopWatch stopWatch = new StopWatch();
     stopWatch.start();
     awsSecretsManagerUtil =
-        new AwsSecretsManagerUtil(AWS_REGION, RW_AWS_ACCESS_KEY_ID, RW_AWS_SECRET_ACCESS_KEY);
+        new AwsSecretsManagerUtil(
+            AWS_REGION, RW_AWS_ACCESS_KEY_ID, RW_AWS_SECRET_ACCESS_KEY, awsEndpointOverride);
     final SecureRandom secureRandom = new SecureRandom();
     LOG.info("Creating {} AWS keys in Secrets Manager in {}", NUMBER_OF_KEYS, AWS_REGION);
     blsKeyPairs =
@@ -119,6 +127,7 @@ void largeNumberOfKeysAreLoadedSuccessfully() {
             .withAccessKeyId(RO_AWS_ACCESS_KEY_ID)
             .withSecretAccessKey(RO_AWS_SECRET_ACCESS_KEY)
             .withPrefixesFilter(List.of(awsSecretsManagerUtil.getSecretsManagerPrefix()))
+            .withEndpointOverride(awsEndpointOverride)
             .build();
 
     final SignerConfigurationBuilder configBuilder =

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AzureKeyVaultAcceptanceTest.java

@@ -13,6 +13,7 @@
 package tech.pegasys.web3signer.tests.bulkloading;
 
 import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 
 import tech.pegasys.web3signer.dsl.signer.SignerConfigurationBuilder;
@@ -58,6 +59,13 @@ void ensureSecretsInKeyVaultAreLoadedAndReportedViaPublicKeysApi() {
 
     final Response response = signer.callApiPublicKeys(KeyType.BLS);
     response.then().statusCode(200).contentType(ContentType.JSON).body("", contains(EXPECTED_KEY));
+
+    signer
+        .healthcheck()
+        .then()
+        .statusCode(200)
+        .contentType(ContentType.JSON)
+        .body("status", equalTo("UP"));
   }
 
   @Test
@@ -72,6 +80,13 @@ void invalidVaultParametersFailsToLoadKeys() {
 
     final Response response = signer.callApiPublicKeys(KeyType.BLS);
     response.then().statusCode(200).contentType(ContentType.JSON).body("", hasSize(0));
+
+    signer
+        .healthcheck()
+        .then()
+        .statusCode(503)
+        .contentType(ContentType.JSON)
+        .body("status", equalTo("DOWN"));
   }
 
   @Test

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/publickeys/AwsKeyIdentifiersAcceptanceTest.java

@@ -17,6 +17,7 @@
 
 import tech.pegasys.web3signer.AwsSecretsManagerUtil;
 
+import java.net.URI;
 import java.util.Collections;
 import java.util.Optional;
 
@@ -59,6 +60,12 @@ public class AwsKeyIdentifiersAcceptanceTest extends KeyIdentifiersAcceptanceTes
   private static final String AWS_REGION =
       Optional.ofNullable(System.getenv("AWS_REGION")).orElse("us-east-2");
 
+  // can be pointed to localstack
+  private final Optional<URI> awsEndpointOverride =
+      System.getenv("AWS_ENDPOINT_OVERRIDE") != null
+          ? Optional.of(URI.create(System.getenv("AWS_ENDPOINT_OVERRIDE")))
+          : Optional.empty();
+
   private final String privateKey = privateKeys(BLS)[0]; // secret value
   private final String publicKey = BLS_PUBLIC_KEY_1;
 
@@ -67,7 +74,8 @@ public class AwsKeyIdentifiersAcceptanceTest extends KeyIdentifiersAcceptanceTes
   @BeforeAll
   void setup() {
     awsSecretsManagerUtil =
-        new AwsSecretsManagerUtil(AWS_REGION, RW_AWS_ACCESS_KEY_ID, RW_AWS_SECRET_ACCESS_KEY);
+        new AwsSecretsManagerUtil(
+            AWS_REGION, RW_AWS_ACCESS_KEY_ID, RW_AWS_SECRET_ACCESS_KEY, awsEndpointOverride);
     awsSecretsManagerUtil.createSecret(publicKey, privateKey, Collections.emptyMap());
   }