Skip to content

Commit 9d534ef

Browse files
Luke SikinaLuke-Sikina
Luke Sikina
authored and
Luke-Sikina
committed
[CHIP-1] Add query.json and patients.txt
- Add migrations via flyway - Change db schema for new fields - Add to API to support patient and query upload
1 parent f5cb9a6 commit 9d534ef

16 files changed

+211
-33
lines changed

‎uploader/README.md

+4-8
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ that Amazon provides you.
4747

4848
5. Run
4949

50-
Run `docker run --rm --env-file .env gic-data-uploader`. Here is an example of a
50+
Run `docker compose --profile production up -d `. Here is an example of a
5151
successful output, with logging prefixes omitted:
5252

5353
```
@@ -79,10 +79,6 @@ Verifying delete capabilities
7979
S3 connection verified.
8080
```
8181

82-
TODO:
83-
- Mount this study in service workbench
84-
- Admin UI
85-
- Test all 4 workspace types
86-
- Mount study
87-
- R/W access
88-
-
82+
6. Run Migrations
83+
84+
`docker compose --profile migrate up -d`

‎uploader/docker-compose.yml

+16-3
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,9 @@ services:
1414
volumes:
1515
- $DOCKER_CONFIG_DIR/aws_uploads/:/gic_query_results/
1616
networks:
17-
- hpdsNet
1817
- picsure
18+
profiles:
19+
- "production"
1920
uploader-db:
2021
image: mysql:8.0
2122
container_name: uploader-db
@@ -39,11 +40,23 @@ services:
3940
volumes:
4041
- /usr/local/pic-sure-services/uploader/data/seed.sql:/docker-entrypoint-initdb.d/seed.sql:ro
4142
- uploader-db-data:/var/lib/mysql
43+
migrations:
44+
image: flyway/flyway:11-alpine
45+
container_name: flyway
46+
command: -url=jdbc:mysql://uploader-db:3306/${DATA_UPLOAD_DB_DATABASE}?allowPublicKeyRetrieval=true -schemas=${DATA_UPLOAD_DB_DATABASE} -user=${DATA_UPLOAD_DB_USER} -password=${DATA_UPLOAD_DB_PASS} -connectRetries=60 -validateMigrationNaming=true migrate
47+
env_file:
48+
- .env
49+
volumes:
50+
- ./flyway:/flyway/sql
51+
networks:
52+
- picsure
53+
profiles:
54+
- "migrate"
55+
depends_on:
56+
- "uploader-db"
4257
volumes:
4358
uploader-db-data:
4459

4560
networks:
4661
picsure:
4762
external: true
48-
hpdsNet:
49-
external: true

‎uploader/flyway/V1__initial.sql

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
CREATE TABLE IF NOT EXISTS `query_status` (
2+
QUERY BINARY(16) NOT NULL,
3+
GENOMIC_STATUS VARCHAR(64) NOT NULL DEFAULT 'Unsent',
4+
PHENOTYPIC_STATUS VARCHAR(64) NOT NULL DEFAULT 'Unsent',
5+
APPROVED DATETIME,
6+
SITE VARCHAR(64) NOT NULL DEFAULT '',
7+
PRIMARY KEY (`QUERY`)
8+
);

‎uploader/flyway/V2__add_patient.sql

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
ALTER TABLE `query_status`
2+
ADD PATIENT_STATUS VARCHAR(64) NOT NULL DEFAULT 'Unsent';
3+
4+
ALTER TABLE `query_status`
5+
ADD QUERY_JSON_STATUS VARCHAR(64) NOT NULL DEFAULT 'Unsent';

‎uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSClient.java

+4
Original file line numberDiff line numberDiff line change
@@ -105,4 +105,8 @@ private String createBody(Object query) {
105105
return null;
106106
}
107107
}
108+
109+
public boolean writePatientData(Query query) {
110+
return writeData(query, "patients");
111+
}
108112
}

‎uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/DataUploadStatuses.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import java.time.LocalDate;
66

77
public record DataUploadStatuses(
8-
UploadStatus genomic, UploadStatus phenotypic, String queryId, @Nullable LocalDate approved, String site
8+
UploadStatus genomic, UploadStatus phenotypic, UploadStatus patient, UploadStatus query,
9+
String queryId, @Nullable LocalDate approved, String site
910
) {
1011
}

‎uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/DataUploadStatusesMapper.java

+5-3
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,15 @@
1212
public class DataUploadStatusesMapper implements RowMapper<DataUploadStatuses> {
1313
@Override
1414
public DataUploadStatuses mapRow(ResultSet rs, int rowNum) throws SQLException {
15-
UploadStatus genomicStatus = UploadStatus.fromString(rs.getString("GENOMIC_STATUS"));
16-
UploadStatus phenotypicStatus = UploadStatus.fromString(rs.getString("PHENOTYPIC_STATUS"));
15+
UploadStatus genomic = UploadStatus.fromString(rs.getString("GENOMIC_STATUS"));
16+
UploadStatus pheno = UploadStatus.fromString(rs.getString("PHENOTYPIC_STATUS"));
17+
UploadStatus patient = UploadStatus.fromString(rs.getString("PATIENT_STATUS"));
18+
UploadStatus queryStatus = UploadStatus.fromString(rs.getString("QUERY_JSON_STATUS"));
1719
String query = fromDashlessString(rs.getString("QUERY")).toString();
1820
Date approved = rs.getDate("APPROVED");
1921
String site = rs.getString("SITE");
2022
return new DataUploadStatuses(
21-
genomicStatus, phenotypicStatus, query, approved == null ? null : approved.toLocalDate(), site
23+
genomic, pheno, patient, queryStatus, query, approved == null ? null : approved.toLocalDate(), site
2224
);
2325
}
2426

‎uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/StatusRepository.java

+23-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package edu.harvard.dbmi.avillach.dataupload.status;
22

3+
import edu.harvard.dbmi.avillach.dataupload.hpds.hpdsartifactsdonotchange.Query;
34
import org.springframework.beans.factory.annotation.Autowired;
45
import org.springframework.jdbc.core.JdbcTemplate;
56
import org.springframework.stereotype.Repository;
@@ -18,7 +19,8 @@ public class StatusRepository {
1819
public Optional<DataUploadStatuses> getQueryStatus(String queryId) {
1920
String sql = """
2021
SELECT
21-
GENOMIC_STATUS, PHENOTYPIC_STATUS, hex(QUERY) as QUERY, APPROVED, SITE
22+
GENOMIC_STATUS, PHENOTYPIC_STATUS, PATIENT_STATUS, QUERY_JSON_STATUS,
23+
hex(QUERY) as QUERY, APPROVED, SITE
2224
FROM
2325
query_status
2426
WHERE
@@ -68,4 +70,24 @@ public void setSite(String picSureId, String site) {
6870
""";
6971
template.update(sql, picSureId.replace("-", ""), site, site);
7072
}
73+
74+
public void setPatientStatus(String queryId, UploadStatus status) {
75+
String sql = """
76+
INSERT INTO query_status
77+
(query, patient_status)
78+
VALUES (unhex(?), ?)
79+
ON DUPLICATE KEY UPDATE patient_status=?
80+
""";
81+
template.update(sql, queryId.replace("-", ""), status.toString(), status.toString());
82+
}
83+
84+
public void setQueryUploadStatus(String queryId, UploadStatus status) {
85+
String sql = """
86+
INSERT INTO query_status
87+
(query, query_json_status)
88+
VALUES (unhex(?), ?)
89+
ON DUPLICATE KEY UPDATE query_json_status=?
90+
""";
91+
template.update(sql, queryId.replace("-", ""), status.toString(), status.toString());
92+
}
7193
}

‎uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/StatusService.java

+8
Original file line numberDiff line numberDiff line change
@@ -44,4 +44,12 @@ public Optional<DataUploadStatuses> approve(String queryId, LocalDate approvalDa
4444
public void setSite(Query query, String site) {
4545
repository.setSite(query.getPicSureId(), site);
4646
}
47+
48+
public void setPatientStatus(Query query, UploadStatus uploadStatus) {
49+
repository.setPatientStatus(query.getPicSureId(), uploadStatus);
50+
}
51+
52+
public void setQueryUploadStatus(Query query, UploadStatus uploadStatus) {
53+
repository.setQueryUploadStatus(query.getPicSureId(), uploadStatus);
54+
}
4755
}

‎uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/DataType.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import java.util.function.Function;
1010

1111
public enum DataType {
12-
Genomic("genomic_data.tsv"), Phenotypic("phenotypic_data.csv");
12+
Genomic("genomic_data.tsv"), Phenotypic("phenotypic_data.csv"), Patient("patients.txt");
1313
public final String fileName;
1414

1515
DataType(String fileName) {
@@ -20,13 +20,15 @@ public BiConsumer<Query, UploadStatus> getStatusSetter(StatusService statusServi
2020
return switch (this) {
2121
case Genomic -> statusService::setGenomicStatus;
2222
case Phenotypic -> statusService::setPhenotypicStatus;
23+
case Patient -> statusService::setPatientStatus;
2324
};
2425
}
2526

2627
public Function<Query, Boolean> getHPDSUpload(HPDSClient client) {
2728
return switch (this) {
2829
case Genomic -> client::writeGenomicData;
2930
case Phenotypic -> client::writePhenotypicData;
31+
case Patient -> client::writePatientData;
3032
};
3133
}
3234
}

‎uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/DataUploadService.java

+35
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package edu.harvard.dbmi.avillach.dataupload.upload;
22

3+
import com.fasterxml.jackson.core.JsonProcessingException;
4+
import com.fasterxml.jackson.databind.ObjectMapper;
35
import edu.harvard.dbmi.avillach.dataupload.aws.AWSClientBuilder;
46
import edu.harvard.dbmi.avillach.dataupload.aws.SiteAWSInfo;
57
import edu.harvard.dbmi.avillach.dataupload.hpds.HPDSClient;
@@ -62,6 +64,8 @@ public class DataUploadService {
6264
@Autowired
6365
private Map<String, SiteAWSInfo> roleARNs;
6466

67+
private static final ObjectMapper mapper = new ObjectMapper();
68+
6569
public DataUploadStatuses asyncUpload(Query query, String site, DataType dataType) {
6670
dataType.getStatusSetter(statusService).accept(query, UploadStatus.Queued);
6771
Thread.ofVirtual().start(() -> uploadData(query, dataType, site));
@@ -110,10 +114,41 @@ protected void uploadData(Query query, DataType dataType, String site) {
110114
} else {
111115
statusSetter.accept(query, UploadStatus.Error);
112116
}
117+
uploadQueryJson(query, roleARNs.get(site));
113118
LOG.info("Releasing lock for {} / {}", dataType, query.getPicSureId());
114119
uploadLock.release();
115120
}
116121

122+
private void uploadQueryJson(Query query, SiteAWSInfo site) {
123+
UploadStatus queryUploadStatus = statusService.getStatus(query.getPicSureId())
124+
.map(DataUploadStatuses::query)
125+
.orElse(UploadStatus.Unsent);
126+
if (queryUploadStatus == UploadStatus.Uploaded || queryUploadStatus == UploadStatus.Uploading) {
127+
return;
128+
}
129+
statusService.setQueryUploadStatus(query, UploadStatus.Uploading);
130+
LOG.info("Uploading query json for {}", query.getPicSureId());
131+
try {
132+
String queryJson = mapper.writeValueAsString(query);
133+
LOG.info("Created query JSON. Writing to file.");
134+
Path jsonPath = Path.of(sharingRoot.toString(), query.getPicSureId(), "query.json");
135+
Files.writeString(jsonPath, queryJson);
136+
if (!uploadFileFromPath(jsonPath, site, query.getPicSureId())) {
137+
LOG.info("Failed to write query.json");
138+
statusService.setQueryUploadStatus(query, UploadStatus.Error);
139+
}
140+
Files.delete(jsonPath);
141+
} catch (JsonProcessingException e) {
142+
statusService.setQueryUploadStatus(query, UploadStatus.Error);
143+
LOG.info("Failed to get query json: ", e);
144+
} catch (IOException e) {
145+
statusService.setQueryUploadStatus(query, UploadStatus.Error);
146+
LOG.info("Failed to write query json: ", e);
147+
}
148+
LOG.info("Successfully uploaded query.json for {} to {}", query.getPicSureId(), site.siteName());
149+
statusService.setQueryUploadStatus(query, UploadStatus.Uploaded);
150+
}
151+
117152
private void deleteFile(Path data) {
118153
try {
119154
Files.delete(data);

‎uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/status/StatusRepositoryTest.java

+28-2
Original file line numberDiff line numberDiff line change
@@ -129,8 +129,8 @@ void shouldGetQueryId() {
129129

130130
DataUploadStatuses actual = subject.getQueryStatus(query.getPicSureId()).orElseThrow();
131131
DataUploadStatuses expected = new DataUploadStatuses(
132-
UploadStatus.Uploaded, UploadStatus.Error, "33613336-3934-3761-2d38-3233312d3131",
133-
LocalDate.of(2022, 2, 22), "bch"
132+
UploadStatus.Uploaded, UploadStatus.Error, UploadStatus.Unsent, UploadStatus.Uploading,
133+
"33613336-3934-3761-2d38-3233312d3131", LocalDate.of(2022, 2, 22), "bch"
134134
);
135135

136136
Assertions.assertEquals(expected, actual);
@@ -148,4 +148,30 @@ void shouldSetSite() {
148148

149149
Assertions.assertEquals(expected, actual);
150150
}
151+
152+
@Test
153+
void shouldSetPatient() {
154+
Query query = new Query();
155+
query.setPicSureId(UUID.fromString("33613336-3934-3761-2d38-3233312d3131").toString());
156+
157+
subject.setPatientStatus(query.getPicSureId(), UploadStatus.Uploaded);
158+
Optional<UploadStatus> actual = subject.getQueryStatus(query.getPicSureId())
159+
.map(DataUploadStatuses::patient);
160+
Optional<UploadStatus> expected = Optional.of(UploadStatus.Uploaded);
161+
162+
Assertions.assertEquals(expected, actual);
163+
}
164+
165+
@Test
166+
void shouldSetQueryStatus() {
167+
Query query = new Query();
168+
query.setPicSureId(UUID.fromString("33613336-3934-3761-2d38-3233312d3131").toString());
169+
170+
subject.setQueryUploadStatus(query.getPicSureId(), UploadStatus.Error);
171+
Optional<UploadStatus> actual = subject.getQueryStatus(query.getPicSureId())
172+
.map(DataUploadStatuses::query);
173+
Optional<UploadStatus> expected = Optional.of(UploadStatus.Error);
174+
175+
Assertions.assertEquals(expected, actual);
176+
}
151177
}

‎uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/status/StatusServiceTest.java

+24-2
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,10 @@ void shouldApprove() {
7575
void shouldGetQueryStatus() {
7676
Query q = new Query();
7777
q.setPicSureId(":)");
78-
DataUploadStatuses statuses =
79-
new DataUploadStatuses(UploadStatus.Error, UploadStatus.Error, ":)", LocalDate.now(), "bch");
78+
DataUploadStatuses statuses = new DataUploadStatuses(
79+
UploadStatus.Error, UploadStatus.Error, UploadStatus.Unsent, UploadStatus.Unsent,
80+
":)", LocalDate.now(), "bch"
81+
);
8082
Mockito.when(repository.getQueryStatus(":)"))
8183
.thenReturn(Optional.of(statuses));
8284

@@ -85,4 +87,24 @@ void shouldGetQueryStatus() {
8587

8688
Assertions.assertEquals(expected, actual);
8789
}
90+
91+
@Test
92+
void shouldSetPatientStatus() {
93+
Query q = new Query();
94+
q.setPicSureId(":)");
95+
96+
subject.setPatientStatus(q, UploadStatus.Uploading);
97+
98+
Mockito.verify(repository, Mockito.times(1)).setPatientStatus(":)", UploadStatus.Uploading);
99+
}
100+
101+
@Test
102+
void shouldSetQueryStatus() {
103+
Query q = new Query();
104+
q.setPicSureId(":)");
105+
106+
subject.setQueryUploadStatus(q, UploadStatus.Uploading);
107+
108+
Mockito.verify(repository, Mockito.times(1)).setQueryUploadStatus(":)", UploadStatus.Uploading);
109+
}
88110
}

‎uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/upload/DataUploadControllerTest.java

+11-7
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,14 @@ public void init() {
4040
void shouldUpload() {
4141
Query query = new Query();
4242
query.setPicSureId("my id");
43-
DataUploadStatuses before =
44-
new DataUploadStatuses(UploadStatus.Unsent, UploadStatus.Unsent, query.getPicSureId(), LocalDate.EPOCH, "bch");
45-
DataUploadStatuses after =
46-
new DataUploadStatuses(UploadStatus.Uploading, UploadStatus.Uploading, query.getPicSureId(), LocalDate.EPOCH, "bch");
43+
DataUploadStatuses before = new DataUploadStatuses(
44+
UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent,
45+
query.getPicSureId(), LocalDate.EPOCH, "bch"
46+
);
47+
DataUploadStatuses after = new DataUploadStatuses(
48+
UploadStatus.Uploading, UploadStatus.Uploading, UploadStatus.Unsent, UploadStatus.Unsent,
49+
query.getPicSureId(), LocalDate.EPOCH, "bch"
50+
);
4751
Mockito.when(statusService.getStatus(query.getPicSureId()))
4852
.thenReturn(Optional.of(before));
4953
Mockito.when(uploadService.asyncUpload(query, "bch", DataType.Genomic))
@@ -85,7 +89,7 @@ void shouldBlockUnapproved() {
8589
Query query = new Query();
8690
query.setPicSureId("my id");
8791
DataUploadStatuses nullApprovalDate =
88-
new DataUploadStatuses(UploadStatus.Unsent, UploadStatus.Unsent, query.getPicSureId(), null, "bch");
92+
new DataUploadStatuses(UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, query.getPicSureId(), null, "bch");
8993
Mockito.when(statusService.getStatus(query.getPicSureId()))
9094
.thenReturn(Optional.of(nullApprovalDate));
9195

@@ -99,7 +103,7 @@ void shouldBlockApprovedInFuture() {
99103
Query query = new Query();
100104
query.setPicSureId("my id");
101105
DataUploadStatuses nullApprovalDate =
102-
new DataUploadStatuses(UploadStatus.Unsent, UploadStatus.Unsent, query.getPicSureId(), LocalDate.MAX, "bch");
106+
new DataUploadStatuses(UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, query.getPicSureId(), LocalDate.MAX, "bch");
103107
Mockito.when(statusService.getStatus(query.getPicSureId()))
104108
.thenReturn(Optional.of(nullApprovalDate));
105109

@@ -113,7 +117,7 @@ void shouldNoOpWhenAlreadyUploading() {
113117
Query query = new Query();
114118
query.setPicSureId("my id");
115119
DataUploadStatuses uploading =
116-
new DataUploadStatuses(UploadStatus.Uploading, UploadStatus.Uploading, query.getPicSureId(), LocalDate.EPOCH, "bch");
120+
new DataUploadStatuses(UploadStatus.Uploading, UploadStatus.Uploading, UploadStatus.Unsent, UploadStatus.Unsent, query.getPicSureId(), LocalDate.EPOCH, "bch");
117121
Mockito.when(statusService.getStatus(query.getPicSureId()))
118122
.thenReturn(Optional.of(uploading));
119123

0 commit comments

Comments
 (0)