Skip to content

Commit 055b705

Browse files
author
Luke Sikina
committed
[ALS-7539] Test HPDS connection
1 parent daf9d4a commit 055b705

File tree

7 files changed

+213
-4
lines changed

7 files changed

+213
-4
lines changed

‎uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/S3StateVerifier.java

+12-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package edu.harvard.dbmi.avillach.dataupload.aws;
22

3+
import edu.harvard.dbmi.avillach.dataupload.hpds.HPDSConnectionVerifier;
34
import jakarta.annotation.PostConstruct;
45
import org.slf4j.Logger;
56
import org.slf4j.LoggerFactory;
@@ -24,16 +25,23 @@ public class S3StateVerifier {
2425
private static final String testTempPrefix = "S3_DAEMON_INIT_TEST";
2526
private static final Logger LOG = LoggerFactory.getLogger(S3StateVerifier.class);
2627

27-
@Autowired
28-
private Map<String, SiteAWSInfo> sites;
28+
private final Map<String, SiteAWSInfo> sites;
29+
private final AWSClientBuilder clientBuilder;
30+
private final HPDSConnectionVerifier hpdsVerifier;
2931

3032
@Autowired
31-
private AWSClientBuilder clientBuilder;
33+
public S3StateVerifier(Map<String, SiteAWSInfo> sites, AWSClientBuilder clientBuilder, HPDSConnectionVerifier hpdsVerifier) {
34+
this.sites = sites;
35+
this.clientBuilder = clientBuilder;
36+
this.hpdsVerifier = hpdsVerifier;
37+
}
3238

3339
@PostConstruct
3440
private void verifyS3Status() {
3541
sites.values().forEach(inst -> Thread.ofVirtual().start(() -> asyncVerify(inst)));
36-
42+
if (!hpdsVerifier.verifyConnection()) {
43+
throw new RuntimeException("Not correctly connected to HPDS");
44+
}
3745
}
3846

3947
private void asyncVerify(SiteAWSInfo institution) {

‎uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSClient.java

+4
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,10 @@ public class HPDSClient {
3333
@Autowired
3434
private HttpClientContext context;
3535

36+
public boolean writeTestData(Query query) {
37+
return writeData(query, "test_upload");
38+
}
39+
3640
public boolean writePhenotypicData(Query query) {
3741
return writeData(query, "phenotypic");
3842
}

‎uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSConnectionVerifier.java

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
package edu.harvard.dbmi.avillach.dataupload.hpds;
2+
3+
import edu.harvard.dbmi.avillach.dataupload.hpds.hpdsartifactsdonotchange.Query;
4+
import edu.harvard.dbmi.avillach.dataupload.hpds.hpdsartifactsdonotchange.ResultType;
5+
import org.slf4j.Logger;
6+
import org.slf4j.LoggerFactory;
7+
import org.springframework.beans.factory.annotation.Autowired;
8+
import org.springframework.stereotype.Service;
9+
10+
import java.io.File;
11+
import java.nio.file.Path;
12+
13+
@Service
14+
public class HPDSConnectionVerifier {
15+
16+
private static final Logger log = LoggerFactory.getLogger(HPDSConnectionVerifier.class);
17+
18+
private final HPDSClient client;
19+
private final Path sharingRoot;
20+
private final UUIDGenerator uuidGenerator;
21+
22+
@Autowired
23+
public HPDSConnectionVerifier(HPDSClient client, Path sharingRoot, UUIDGenerator uuidGenerator) {
24+
this.client = client;
25+
this.sharingRoot = sharingRoot;
26+
this.uuidGenerator = uuidGenerator;
27+
}
28+
29+
public boolean verifyConnection() {
30+
log.info("Verifying connection to hpds by asking it to create a file and then verifying that the file exists.");
31+
Query testQuery = new Query();
32+
testQuery.setPicSureId(uuidGenerator.generate().toString());
33+
testQuery.setId(testQuery.getPicSureId());
34+
testQuery.setExpectedResultType(ResultType.COUNT);
35+
log.info("Created test query with UUID {}", testQuery.getPicSureId());
36+
37+
log.info("Sending test query to HPDS");
38+
boolean hpdsResponse = client.writeTestData(testQuery);
39+
if (!hpdsResponse) {
40+
log.info("HPDS returned non 200 exit code. Assuming failure.");
41+
return false;
42+
}
43+
log.info("HPDS reported successfully writing the data. Verifying that the file exists");
44+
File testData = Path.of(sharingRoot.toString(), testQuery.getPicSureId(), "test_data.txt").toFile();
45+
46+
if (testData.exists() && testData.isFile()) {
47+
log.info("File found! Connection to HPDS verified!");
48+
return testData.delete();
49+
}
50+
log.info(
51+
"File {} not found. HPDS is running, but the shared directory is probably misconfigured",
52+
testData.getPath()
53+
);
54+
return false;
55+
}
56+
}

‎uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/UUIDGenerator.java

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package edu.harvard.dbmi.avillach.dataupload.hpds;
2+
3+
import org.springframework.stereotype.Service;
4+
5+
import java.util.UUID;
6+
7+
@Service
8+
public class UUIDGenerator {
9+
10+
public UUID generate() {
11+
return UUID.randomUUID();
12+
}
13+
}

‎uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/hpdsartifactsdonotchange/Query.java

+11
Original file line numberDiff line numberDiff line change
@@ -150,5 +150,16 @@ public VariantInfoFilter(VariantInfoFilter filter) {
150150
public Map<String, String[]> categoryVariantInfoFilters;
151151
}
152152

153+
@Override
154+
public boolean equals(Object object) {
155+
if (this == object) return true;
156+
if (object == null || getClass() != object.getClass()) return false;
157+
Query query = (Query) object;
158+
return getExpectedResultType() == query.getExpectedResultType() && Objects.equals(getId(), query.getId()) && Objects.equals(getPicSureId(), query.getPicSureId());
159+
}
153160

161+
@Override
162+
public int hashCode() {
163+
return Objects.hash(getExpectedResultType(), getId(), getPicSureId());
164+
}
154165
}

‎uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSClientTest.java

+17
Original file line numberDiff line numberDiff line change
@@ -117,4 +117,21 @@ void shouldNotWriteGenomicData() throws IOException, InterruptedException {
117117

118118
Assertions.assertFalse(actual);
119119
}
120+
121+
@Test
122+
void shouldWriteTestData() throws IOException {
123+
Query query = new Query();
124+
query.setPicSureId("my id");
125+
126+
Mockito.when(response.getStatusLine())
127+
.thenReturn(line);
128+
Mockito.when(line.getStatusCode())
129+
.thenReturn(200);
130+
Mockito.when(client.execute(Mockito.any(), Mockito.eq(context)))
131+
.thenReturn(response);
132+
133+
boolean actual = subject.writeTestData(query);
134+
135+
Assertions.assertTrue(actual);
136+
}
120137
}

‎uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSConnectionVerifierTest.java

+100
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
package edu.harvard.dbmi.avillach.dataupload.hpds;
2+
3+
import edu.harvard.dbmi.avillach.dataupload.hpds.hpdsartifactsdonotchange.Query;
4+
import edu.harvard.dbmi.avillach.dataupload.hpds.hpdsartifactsdonotchange.ResultType;
5+
import org.junit.jupiter.api.Assertions;
6+
import org.junit.jupiter.api.Test;
7+
import org.junit.jupiter.api.io.TempDir;
8+
import org.mockito.ArgumentMatcher;
9+
import org.mockito.Mockito;
10+
import org.springframework.boot.test.context.SpringBootTest;
11+
import org.springframework.boot.test.mock.mockito.MockBean;
12+
13+
import java.io.IOException;
14+
import java.nio.file.Files;
15+
import java.nio.file.Path;
16+
import java.util.UUID;
17+
18+
import static org.junit.jupiter.api.Assertions.*;
19+
20+
@SpringBootTest
21+
class HPDSConnectionVerifierTest {
22+
23+
@MockBean
24+
private HPDSClient client;
25+
26+
@MockBean
27+
private UUIDGenerator generator;
28+
29+
private final Query query = new Query();
30+
{
31+
query.setPicSureId("9f1fc383-611b-4c6a-af37-a33c07feea5e");
32+
query.setId("9f1fc383-611b-4c6a-af37-a33c07feea5e");
33+
query.setExpectedResultType(ResultType.COUNT);
34+
}
35+
36+
@Test
37+
void shouldFailWhenHPDS400s(@TempDir Path sharingRoot) {
38+
Mockito.when(client.writeTestData(query))
39+
.thenReturn(false);
40+
Mockito.when(generator.generate())
41+
.thenReturn(UUID.fromString("9f1fc383-611b-4c6a-af37-a33c07feea5e"));
42+
HPDSConnectionVerifier subject = new HPDSConnectionVerifier(client, sharingRoot, generator);
43+
44+
boolean result = subject.verifyConnection();
45+
46+
Assertions.assertFalse(result);
47+
Mockito.verify(client, Mockito.times(1))
48+
.writeTestData(query);
49+
}
50+
51+
@Test
52+
void shouldFailWhenHPDSDoesNotWrite(@TempDir Path sharingRoot) {
53+
Mockito.when(client.writeTestData(query))
54+
.thenReturn(true);
55+
Mockito.when(generator.generate())
56+
.thenReturn(UUID.fromString("9f1fc383-611b-4c6a-af37-a33c07feea5e"));
57+
HPDSConnectionVerifier subject = new HPDSConnectionVerifier(client, sharingRoot, generator);
58+
59+
boolean result = subject.verifyConnection();
60+
61+
Assertions.assertFalse(result);
62+
Mockito.verify(client, Mockito.times(1))
63+
.writeTestData(query);
64+
}
65+
66+
@Test
67+
void shouldFailWhenHPDSMakesADirectory(@TempDir Path sharingRoot) throws IOException {
68+
Files.createDirectory(Path.of(sharingRoot.toString(), "9f1fc383-611b-4c6a-af37-a33c07feea5e"));
69+
Files.createDirectory(Path.of(sharingRoot.toString(), "9f1fc383-611b-4c6a-af37-a33c07feea5e", "test_data.txt"));
70+
71+
Mockito.when(client.writeTestData(query))
72+
.thenReturn(true);
73+
Mockito.when(generator.generate())
74+
.thenReturn(UUID.fromString("9f1fc383-611b-4c6a-af37-a33c07feea5e"));
75+
HPDSConnectionVerifier subject = new HPDSConnectionVerifier(client, sharingRoot, generator);
76+
77+
Assertions.assertFalse(subject.verifyConnection());
78+
Mockito.verify(client, Mockito.times(1))
79+
.writeTestData(query);
80+
}
81+
82+
@Test
83+
void shouldPass(@TempDir Path sharingRoot) throws IOException {
84+
Files.createDirectory(Path.of(sharingRoot.toString(), "9f1fc383-611b-4c6a-af37-a33c07feea5e"));
85+
Files.writeString(
86+
Path.of(sharingRoot.toString(), "9f1fc383-611b-4c6a-af37-a33c07feea5e", "test_data.txt"),
87+
"Howdy :)"
88+
);
89+
90+
Mockito.when(client.writeTestData(query))
91+
.thenReturn(true);
92+
Mockito.when(generator.generate())
93+
.thenReturn(UUID.fromString("9f1fc383-611b-4c6a-af37-a33c07feea5e"));
94+
HPDSConnectionVerifier subject = new HPDSConnectionVerifier(client, sharingRoot, generator);
95+
96+
Assertions.assertTrue(subject.verifyConnection());
97+
Mockito.verify(client, Mockito.times(1))
98+
.writeTestData(query);
99+
}
100+
}

0 commit comments

Comments
 (0)