Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix test performance #886

Merged
merged 5 commits into from
Apr 17, 2024
Merged

Fix test performance #886

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6693e41
Insert data offer id in test case
ununhexium Apr 16, 2024
37496dd
Speedup integration tests
ununhexium Apr 16, 2024
5a02a28
checkstyle
ununhexium Apr 16, 2024
048a7a5
cleanup / review
ununhexium Apr 16, 2024
c68c000
Merge branch 'main' into fix-test-performance
kulgg Apr 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions ...ppers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/utils/UiAssetMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import de.sovity.edc.utils.jsonld.vocab.Prop;
import de.sovity.edc.utils.jsonld.vocab.Prop.SovityDcatExt.HttpDatasourceHints;
import jakarta.json.Json;
import jakarta.json.JsonArray;
import jakarta.json.JsonObject;
import jakarta.json.JsonObjectBuilder;
import jakarta.json.JsonValue;
Expand All @@ -31,9 +30,6 @@

import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;

import static de.sovity.edc.ext.wrapper.api.common.mappers.utils.JsonBuilderUtils.addNonNull;
import static de.sovity.edc.ext.wrapper.api.common.mappers.utils.JsonBuilderUtils.addNonNullArray;
Expand Down
149 changes: 82 additions & 67 deletions tests/src/test/java/de/sovity/edc/e2e/DataSourceParameterizationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,6 @@
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import org.mockserver.integration.ClientAndServer;
import org.mockserver.model.HttpRequest;
import org.mockserver.model.HttpResponse;
Expand All @@ -65,6 +63,7 @@
import java.util.Map;
import java.util.Random;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream;
import javax.annotation.Nullable;

Expand Down Expand Up @@ -105,7 +104,6 @@ class DataSourceParameterizationTest {

private EdcClient providerClient;
private EdcClient consumerClient;
private static final String DATA_OFFER_ID = "my-data-offer-2023-11";

private final int port = getFreePort();
private final String sourcePath = "/source/some/path/";
Expand All @@ -115,8 +113,11 @@ class DataSourceParameterizationTest {
// TODO: remove the test backend dependency?
private ClientAndServer mockServer;

private static final AtomicInteger DATA_OFFER_INDEX = new AtomicInteger(0);

record TestCase(
String name,
String id,
String method,
@Nullable String body,
@Nullable String mediaType,
Expand Down Expand Up @@ -167,6 +168,7 @@ void canUseTheWorkaroundInCustomTransferRequest() {
// arrange
val testCase = new TestCase(
"",
"data-offer-" + DATA_OFFER_INDEX.getAndIncrement(),
HttpMethod.PATCH,
"[]",
"application/json",
Expand All @@ -176,27 +178,27 @@ void canUseTheWorkaroundInCustomTransferRequest() {
val received = new AtomicBoolean(false);
prepareDataTransferBackends(testCase, () -> received.set(true));

createPolicy();
val assetId = createAssetWithParamedMethod(testCase);
val contractDefinitionId = createContractDefinition();
createData(testCase);

// act
val dataOffers = consumerClient.uiApi().getCatalogPageDataOffers(getProtocolEndpoint(providerConnector));
val startNegotiation = initiateNegotiation(dataOffers.get(0), dataOffers.get(0).getContractOffers().get(0));
val negotiation = awaitNegotiationDone(startNegotiation.getContractNegotiationId());

String standardBase = "https://w3id.org/edc/v0.0.1/ns/";
String workaroundBase = "https://sovity.de/workaround/proxy/param/";
var transferRequestJsonLd = Json.createObjectBuilder()
.add(
Prop.Edc.DATA_DESTINATION,
Json.createObjectBuilder(Map.of(
"https://w3id.org/edc/v0.0.1/ns/type", "HttpData",
"https://w3id.org/edc/v0.0.1/ns/baseUrl", destinationUrl,
"https://w3id.org/edc/v0.0.1/ns/method", "PUT",
"https://sovity.de/workaround/proxy/param/pathSegments", testCase.path,
"https://sovity.de/workaround/proxy/param/method", testCase.method,
"https://sovity.de/workaround/proxy/param/queryParams", "filter=a&filter=b&filter=c",
"https://sovity.de/workaround/proxy/param/mediaType", testCase.mediaType,
"https://sovity.de/workaround/proxy/param/body", testCase.body
standardBase + "type", "HttpData",
standardBase + "baseUrl", destinationUrl,
standardBase + "method", "PUT",
workaroundBase + "pathSegments", testCase.path,
workaroundBase + "method", testCase.method,
workaroundBase + "queryParams", "filter=a&filter=b&filter=c",
workaroundBase + "mediaType", testCase.mediaType,
workaroundBase + "body", testCase.body
)).build()
)
.add(Prop.Edc.CTX + "transferType", Json.createObjectBuilder()
Expand All @@ -217,18 +219,25 @@ void canUseTheWorkaroundInCustomTransferRequest() {

// assert
TransferHistoryEntry actual = consumerClient.uiApi().getTransferHistoryPage().getTransferEntries().get(0);
assertThat(actual.getAssetId()).isEqualTo(assetId);
assertThat(actual.getAssetId()).isEqualTo(testCase.id);
assertThat(actual.getTransferProcessId()).isEqualTo(transferId);
assertThat(actual.getState().getSimplifiedState()).isEqualTo(OK);

assertThat(received.get()).isTrue();
}

private void createData(TestCase testCase) {
createPolicy(testCase);
createAssetWithParameterizedMethod(testCase);
createContractDefinition(testCase);
}

@Test
void sendWithEdcManagementApi() {
// arrange
val testCase = new TestCase(
"",
"data-offer-" + DATA_OFFER_INDEX.getAndIncrement(),
HttpMethod.PATCH,
"[]",
"application/json",
Expand All @@ -238,71 +247,77 @@ void sendWithEdcManagementApi() {
val received = new AtomicBoolean(false);
prepareDataTransferBackends(testCase, () -> received.set(true));

createPolicy();
val assetId = createAssetWithParamedMethod(testCase);
createContractDefinition();
createData(testCase);

// act
val dataOffers = consumerClient.uiApi().getCatalogPageDataOffers(getProtocolEndpoint(providerConnector));
val startNegotiation = initiateNegotiation(dataOffers.get(0), dataOffers.get(0).getContractOffers().get(0));
val negotiation = awaitNegotiationDone(startNegotiation.getContractNegotiationId());

String workaroundBase = "https://sovity.de/workaround/proxy/param/";
String standardBase = "https://w3id.org/edc/v0.0.1/ns/";
val transferId = consumerConnector.initiateTransfer(
negotiation.getContractAgreementId(),
assetId,
testCase.id,
URI.create("http://localhost:21003/api/dsp"),
Json.createObjectBuilder(Map.of(
"https://w3id.org/edc/v0.0.1/ns/type", "HttpData",
"https://w3id.org/edc/v0.0.1/ns/baseUrl", destinationUrl,
"https://w3id.org/edc/v0.0.1/ns/method", "PUT",
"https://sovity.de/workaround/proxy/param/pathSegments", testCase.path,
"https://sovity.de/workaround/proxy/param/method", testCase.method,
"https://sovity.de/workaround/proxy/param/queryParams", "filter=a&filter=b&filter=c",
"https://sovity.de/workaround/proxy/param/mediaType", testCase.mediaType,
"https://sovity.de/workaround/proxy/param/body", testCase.body
standardBase + "type", "HttpData",
standardBase + "baseUrl", destinationUrl,
standardBase + "method", "PUT",
workaroundBase + "pathSegments", testCase.path,
workaroundBase + "method", testCase.method,
workaroundBase + "queryParams", "filter=a&filter=b&filter=c",
workaroundBase + "mediaType", testCase.mediaType,
workaroundBase + "body", testCase.body
)).build()
);

awaitTransferCompletion(transferId);

// assert
TransferHistoryEntry actual = consumerClient.uiApi().getTransferHistoryPage().getTransferEntries().get(0);
assertThat(actual.getAssetId()).isEqualTo(assetId);
assertThat(actual.getAssetId()).isEqualTo(testCase.id);
assertThat(actual.getTransferProcessId()).isEqualTo(transferId);
assertThat(actual.getState().getSimplifiedState()).isEqualTo(OK);

assertThat(received.get()).isTrue();
}

@ParameterizedTest(name = "{0}")
@MethodSource("source")
void canTransferParameterizedAsset(TestCase testCase) {
// arrange
val received = new AtomicBoolean(false);
prepareDataTransferBackends(testCase, () -> received.set(true));

createPolicy();
val assetId = createAssetWithParamedMethod(testCase);
createContractDefinition();

// act
var dataOffers = consumerClient.uiApi().getCatalogPageDataOffers(getProtocolEndpoint(providerConnector));
var negotiation = initiateNegotiation(dataOffers.get(0), dataOffers.get(0).getContractOffers().get(0));
negotiation = awaitNegotiationDone(negotiation.getContractNegotiationId());
val transferId = initiateTransferWithParameters(negotiation, testCase);

awaitTransferCompletion(transferId);

// assert
TransferHistoryEntry actual = consumerClient.uiApi().getTransferHistoryPage().getTransferEntries().get(0);
assertThat(actual.getAssetId()).isEqualTo(assetId);
assertThat(actual.getTransferProcessId()).isEqualTo(transferId);
assertThat(actual.getState().getSimplifiedState()).isEqualTo(OK);

assertThat(received.get()).isTrue();
@Test
void canTransferParameterizedAsset() {
source().forEach(testCase -> {
// arrange
val received = new AtomicBoolean(false);
prepareDataTransferBackends(testCase, () -> received.set(true));

createData(testCase);

// act
val dataOffers = consumerClient.uiApi().getCatalogPageDataOffers(getProtocolEndpoint(providerConnector));
val dataOffer = dataOffers.stream().filter(it -> it.getAsset().getAssetId().equals(testCase.id)).findFirst().get();
val negotiationInit = initiateNegotiation(dataOffer, dataOffer.getContractOffers().get(0));
val negotiation = awaitNegotiationDone(negotiationInit.getContractNegotiationId());
val transferId = initiateTransferWithParameters(negotiation, testCase);

awaitTransferCompletion(transferId);

// assert
TransferHistoryEntry actual = consumerClient.uiApi()
.getTransferHistoryPage()
.getTransferEntries()
.stream()
.filter(it -> it.getAssetId().equals(testCase.id))
.findFirst()
.get();
assertThat(actual.getAssetId()).isEqualTo(testCase.id);
assertThat(actual.getTransferProcessId()).isEqualTo(transferId);
assertThat(actual.getState().getSimplifiedState()).isEqualTo(OK);

assertThat(received.get()).isTrue();
});
}

private static Stream<TestCase> source() {
private Stream<TestCase> source() {
val httpMethods = List.of(
HttpMethod.POST,
// HttpMethod.HEAD,
Expand All @@ -328,6 +343,7 @@ private static Stream<TestCase> source() {
queryParameters.stream().map(params ->
new TestCase(
method + " body:" + body + " path:" + usePath + " params=" + params,
"data-offer-" + DATA_OFFER_INDEX.getAndIncrement(),
method,
body,
body == null ? null : "application/json",
Expand All @@ -352,9 +368,9 @@ private static List<String> getBodyOptionsFor(String method) {
return useBodyChoices;
}

@NotNull
private void prepareDataTransferBackends(TestCase testCase, Runnable onRequestReceived) {
String payload = generateRandomPayload();
mockServer.reset();

val requestDefinition = request(sourcePath).withMethod(testCase.method);
if (testCase.body != null) {
Expand Down Expand Up @@ -394,11 +410,10 @@ private void prepareDataTransferBackends(TestCase testCase, Runnable onRequestRe
private static String generateRandomPayload() {
byte[] data = new byte[10];
new Random().nextBytes(data);
String payload = Base64.getEncoder().encodeToString(data);
return payload;
return Base64.getEncoder().encodeToString(data);
}

private String createAssetWithParamedMethod(TestCase testCase) {
private String createAssetWithParameterizedMethod(TestCase testCase) {
val proxyProperties = new HashMap<>(Map.of(
Prop.Edc.TYPE, "HttpData",
Prop.Edc.BASE_URL, sourceUrl
Expand All @@ -417,17 +432,17 @@ private String createAssetWithParamedMethod(TestCase testCase) {
}

var asset = UiAssetCreateRequest.builder()
.id(DATA_OFFER_ID)
.id(testCase.id)
.title("My Data Offer")
.dataAddressProperties(proxyProperties)
.build();

return providerClient.uiApi().createAsset(asset).getId();
}

private void createPolicy() {
private void createPolicy(TestCase testCase) {
var policyDefinition = PolicyDefinitionCreateRequest.builder()
.policyDefinitionId(DATA_OFFER_ID)
.policyDefinitionId(testCase.id)
.policy(UiPolicyCreateRequest.builder()
.constraints(List.of())
.build())
Expand All @@ -436,17 +451,17 @@ private void createPolicy() {
providerClient.uiApi().createPolicyDefinition(policyDefinition);
}

private String createContractDefinition() {
private String createContractDefinition(TestCase testCase) {
var contractDefinition = ContractDefinitionRequest.builder()
.contractDefinitionId(DATA_OFFER_ID)
.accessPolicyId(DATA_OFFER_ID)
.contractPolicyId(DATA_OFFER_ID)
.contractDefinitionId(testCase.id)
.accessPolicyId(testCase.id)
.contractPolicyId(testCase.id)
.assetSelector(List.of(UiCriterion.builder()
.operandLeft(Prop.Edc.ID)
.operator(UiCriterionOperator.EQ)
.operandRight(UiCriterionLiteral.builder()
.type(UiCriterionLiteralType.VALUE)
.value(DATA_OFFER_ID)
.value(testCase.id)
.build())
.build()))
.build();
Expand Down
Loading