Skip to content

Commit 331dcf7

Browse files
Gcolon021Luke Sikina
Gcolon021
authored and
Luke Sikina
committed
[ALS-7809] DRS URI: Data Dictionary API (#56)
* Add new endpoint that excepts a list of concepts Returns a list of concepts with their metadata included.
1 parent f76241d commit 331dcf7

File tree

9 files changed

+186
-7
lines changed

9 files changed

+186
-7
lines changed

‎dictonaryReqeust.http

+7
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,10 @@ Content-Type: application/json
1414

1515
{"@type":"GeneralQueryRequest","resourceCredentials":{},"query":{"searchTerm":"throat sore acute #8","includedTags":[],
1616
"excludedTags":[],"returnTags":"true","offset":0,"limit":100000},"resourceUUID":null}
17+
18+
###
19+
20+
POST http://localhost:80/concepts/detail
21+
Content-Type: application/json
22+
23+
["\\phs000993\\pht005015\\phv00253191\\BODY_SITE\\", "\\phs002913\\W2Q_COV_REINFEC_2_OTH\\"]

‎src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptController.java

+5
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,11 @@ public ResponseEntity<Concept> conceptDetail(@PathVariable(name = "dataset") Str
5858
return conceptService.conceptDetail(dataset, conceptPath).map(ResponseEntity::ok).orElse(ResponseEntity.notFound().build());
5959
}
6060

61+
@PostMapping(path = "/concepts/detail")
62+
public ResponseEntity<List<Concept>> conceptsDetail(@RequestBody() List<String> conceptPaths) {
63+
return ResponseEntity.ok(conceptService.conceptsWithDetail(conceptPaths));
64+
}
65+
6166
@PostMapping(path = "/concepts/tree/{dataset}")
6267
public ResponseEntity<Concept> conceptTree(
6368
@PathVariable(name = "dataset") String dataset, @RequestBody() String conceptPath,

‎src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java

+51-2
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept;
44
import edu.harvard.dbmi.avillach.dictionary.filter.Filter;
55
import edu.harvard.dbmi.avillach.dictionary.filter.QueryParamPair;
6-
import edu.harvard.dbmi.avillach.dictionary.legacysearch.SearchResultRowMapper;
76
import edu.harvard.dbmi.avillach.dictionary.util.MapExtractor;
87
import org.springframework.beans.factory.annotation.Autowired;
98
import org.springframework.beans.factory.annotation.Value;
@@ -27,19 +26,21 @@ public class ConceptRepository {
2726
private final ConceptFilterQueryGenerator filterGen;
2827
private final ConceptMetaExtractor conceptMetaExtractor;
2928
private final ConceptResultSetExtractor conceptResultSetExtractor;
29+
private final ConceptRowWithMetaMapper conceptRowWithMetaMapper;
3030
private final List<String> disallowedMetaFields;
3131

3232
@Autowired
3333
public ConceptRepository(
3434
NamedParameterJdbcTemplate template, ConceptRowMapper mapper, ConceptFilterQueryGenerator filterGen,
3535
ConceptMetaExtractor conceptMetaExtractor, ConceptResultSetExtractor conceptResultSetExtractor,
36-
@Value("${filtering.unfilterable_concepts}") List<String> disallowedMetaFields
36+
ConceptRowWithMetaMapper conceptRowWithMetaMapper, @Value("${filtering.unfilterable_concepts}") List<String> disallowedMetaFields
3737
) {
3838
this.template = template;
3939
this.mapper = mapper;
4040
this.filterGen = filterGen;
4141
this.conceptMetaExtractor = conceptMetaExtractor;
4242
this.conceptResultSetExtractor = conceptResultSetExtractor;
43+
this.conceptRowWithMetaMapper = conceptRowWithMetaMapper;
4344
this.disallowedMetaFields = disallowedMetaFields;
4445
}
4546

@@ -230,4 +231,52 @@ WITH RECURSIVE nodes AS (
230231
}
231232

232233

234+
public List<Concept> getConceptsByPathWithMetadata(List<String> conceptPaths) {
235+
String sql = ALLOW_FILTERING_Q + ", "
236+
+ """
237+
filtered_concepts AS (
238+
SELECT
239+
concept_node.*
240+
FROM
241+
concept_node
242+
WHERE
243+
concept_path IN (:conceptPaths)
244+
),
245+
aggregated_meta AS (
246+
SELECT
247+
concept_node_meta.concept_node_id,
248+
json_agg(json_build_object('key', concept_node_meta.key, 'value', concept_node_meta.value)) AS metadata
249+
FROM
250+
concept_node_meta
251+
WHERE
252+
concept_node_meta.concept_node_id IN (
253+
SELECT concept_node_id FROM filtered_concepts
254+
)
255+
GROUP BY
256+
concept_node_meta.concept_node_id
257+
)
258+
SELECT
259+
concept_node.*,
260+
ds.REF as dataset,
261+
ds.abbreviation AS studyAcronym,
262+
continuous_min.VALUE as min, continuous_max.VALUE as max,
263+
categorical_values.VALUE as values,
264+
allow_filtering.allowFiltering AS allowFiltering,
265+
meta_description.VALUE AS description,
266+
aggregated_meta.metadata AS metadata
267+
FROM
268+
filtered_concepts as concept_node
269+
LEFT JOIN dataset AS ds ON concept_node.dataset_id = ds.dataset_id
270+
LEFT JOIN concept_node_meta AS meta_description ON concept_node.concept_node_id = meta_description.concept_node_id AND meta_description.KEY = 'description'
271+
LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min'
272+
LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max'
273+
LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values'
274+
LEFT JOIN allow_filtering ON concept_node.concept_node_id = allow_filtering.concept_node_id
275+
LEFT JOIN aggregated_meta ON concept_node.concept_node_id = aggregated_meta.concept_node_id
276+
""";
277+
278+
MapSqlParameterSource params =
279+
new MapSqlParameterSource().addValue("conceptPaths", conceptPaths).addValue("disallowed_meta_keys", disallowedMetaFields);
280+
return template.query(sql, params, conceptRowWithMetaMapper);
281+
}
233282
}

‎src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java

+19-5
Original file line numberDiff line numberDiff line change
@@ -3,42 +3,56 @@
33
import edu.harvard.dbmi.avillach.dictionary.concept.model.CategoricalConcept;
44
import edu.harvard.dbmi.avillach.dictionary.concept.model.ContinuousConcept;
55
import edu.harvard.dbmi.avillach.dictionary.util.JsonBlobParser;
6-
import org.slf4j.Logger;
7-
import org.slf4j.LoggerFactory;
86
import org.springframework.beans.factory.annotation.Autowired;
97
import org.springframework.stereotype.Component;
108

119
import java.sql.ResultSet;
1210
import java.sql.SQLException;
1311
import java.util.List;
12+
import java.util.Map;
1413

1514
@Component
1615
public class ConceptResultSetUtil {
1716

18-
private static final Logger log = LoggerFactory.getLogger(ConceptResultSetUtil.class);
1917
private final JsonBlobParser jsonBlobParser;
2018

2119
@Autowired
2220
public ConceptResultSetUtil(JsonBlobParser jsonBlobParser) {
2321
this.jsonBlobParser = jsonBlobParser;
2422
}
2523

26-
public CategoricalConcept mapCategorical(ResultSet rs) throws SQLException {
24+
public CategoricalConcept mapCategoricalWithMetadata(ResultSet rs) throws SQLException {
25+
Map<String, String> metadata = jsonBlobParser.parseMetaData(rs.getString("metadata"));
26+
return new CategoricalConcept(getCategoricalConcept(rs), metadata);
27+
}
28+
29+
private CategoricalConcept getCategoricalConcept(ResultSet rs) throws SQLException {
2730
return new CategoricalConcept(
2831
rs.getString("concept_path"), rs.getString("name"), rs.getString("display"), rs.getString("dataset"),
2932
rs.getString("description"), rs.getString("values") == null ? List.of() : jsonBlobParser.parseValues(rs.getString("values")),
3033
rs.getBoolean("allowFiltering"), rs.getString("studyAcronym"), null, null
3134
);
3235
}
3336

34-
public ContinuousConcept mapContinuous(ResultSet rs) throws SQLException {
37+
public ContinuousConcept mapContinuousWithMetadata(ResultSet rs) throws SQLException {
38+
Map<String, String> metadata = jsonBlobParser.parseMetaData(rs.getString("metadata"));
39+
return new ContinuousConcept(getContinuousConcept(rs), metadata);
40+
}
41+
42+
private ContinuousConcept getContinuousConcept(ResultSet rs) throws SQLException {
3543
return new ContinuousConcept(
3644
rs.getString("concept_path"), rs.getString("name"), rs.getString("display"), rs.getString("dataset"),
3745
rs.getString("description"), rs.getBoolean("allowFiltering"), jsonBlobParser.parseMin(rs.getString("values")),
3846
jsonBlobParser.parseMax(rs.getString("values")), rs.getString("studyAcronym"), null
3947
);
4048
}
4149

50+
public ContinuousConcept mapContinuous(ResultSet rs) throws SQLException {
51+
return getContinuousConcept(rs);
52+
}
4253

54+
public CategoricalConcept mapCategorical(ResultSet rs) throws SQLException {
55+
return getCategoricalConcept(rs);
56+
}
4357

4458
}

‎src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRowWithMetaMapper.java

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package edu.harvard.dbmi.avillach.dictionary.concept;
2+
3+
import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept;
4+
import edu.harvard.dbmi.avillach.dictionary.concept.model.ConceptType;
5+
import org.springframework.beans.factory.annotation.Autowired;
6+
import org.springframework.jdbc.core.RowMapper;
7+
import org.springframework.stereotype.Component;
8+
9+
import java.sql.ResultSet;
10+
import java.sql.SQLException;
11+
12+
@Component
13+
public class ConceptRowWithMetaMapper implements RowMapper<Concept> {
14+
15+
private final ConceptResultSetUtil conceptResultSetUtil;
16+
17+
@Autowired
18+
public ConceptRowWithMetaMapper(ConceptResultSetUtil conceptResultSetUtil) {
19+
this.conceptResultSetUtil = conceptResultSetUtil;
20+
}
21+
22+
@Override
23+
public Concept mapRow(ResultSet rs, int rowNum) throws SQLException {
24+
return switch (ConceptType.toConcept(rs.getString("concept_type"))) {
25+
case Categorical -> conceptResultSetUtil.mapCategoricalWithMetadata(rs);
26+
case Continuous -> conceptResultSetUtil.mapContinuousWithMetadata(rs);
27+
};
28+
}
29+
30+
}

‎src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java

+3
Original file line numberDiff line numberDiff line change
@@ -71,4 +71,7 @@ public Optional<Concept> conceptDetailWithoutAncestors(String dataset, String co
7171
return getConcept(dataset, conceptPath, false);
7272
}
7373

74+
public List<Concept> conceptsWithDetail(List<String> conceptPaths) {
75+
return this.conceptRepository.getConceptsByPathWithMetadata(conceptPaths);
76+
}
7477
}

‎src/main/java/edu/harvard/dbmi/avillach/dictionary/util/JsonBlobParser.java

+31
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,31 @@
11
package edu.harvard.dbmi.avillach.dictionary.util;
22

33

4+
import com.fasterxml.jackson.core.JsonProcessingException;
5+
import com.fasterxml.jackson.core.type.TypeReference;
6+
import com.fasterxml.jackson.databind.ObjectMapper;
47
import org.json.JSONArray;
58
import org.json.JSONException;
69
import org.slf4j.Logger;
710
import org.slf4j.LoggerFactory;
811
import org.springframework.stereotype.Component;
12+
import org.springframework.util.StringUtils;
913

1014
import java.math.BigDecimal;
1115
import java.math.BigInteger;
1216
import java.util.ArrayList;
17+
import java.util.HashMap;
1318
import java.util.List;
19+
import java.util.Map;
20+
import java.util.stream.Collectors;
1421

1522
@Component
1623
public class JsonBlobParser {
1724

1825
private final static Logger log = LoggerFactory.getLogger(JsonBlobParser.class);
26+
private final ObjectMapper objectMapper = new ObjectMapper();
27+
28+
public JsonBlobParser() {}
1929

2030
public List<String> parseValues(String valuesArr) {
2131
try {
@@ -62,4 +72,25 @@ public Float parseMax(String valuesArr) {
6272
return parseFromIndex(valuesArr, 1);
6373
}
6474

75+
public Map<String, String> parseMetaData(String jsonMetaData) {
76+
Map<String, String> metadata;
77+
78+
try {
79+
List<Map<String, String>> maps = objectMapper.readValue(jsonMetaData, new TypeReference<List<Map<String, String>>>() {});
80+
// convert the list to a flat map
81+
Map<String, String> map = new HashMap<>();
82+
for (Map<String, String> entry : maps) {
83+
if (map.put(entry.get("key"), entry.get("value")) != null) {
84+
throw new IllegalStateException(
85+
"parseMetaData() Duplicate key found in metadata. Key: " + entry.get("key") + " Value: " + entry.get("value")
86+
);
87+
}
88+
}
89+
metadata = map;
90+
} catch (JsonProcessingException e) {
91+
throw new RuntimeException(e);
92+
}
93+
94+
return metadata;
95+
}
6596
}

‎src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java

+14
Original file line numberDiff line numberDiff line change
@@ -148,4 +148,18 @@ void shouldDumpConcepts() {
148148
Assertions.assertEquals(concepts, actual.getBody().getContent());
149149
Assertions.assertEquals(HttpStatus.OK, actual.getStatusCode());
150150
}
151+
152+
@Test
153+
void shouldReturnConceptsWithMeta() {
154+
CategoricalConcept fooBar = new CategoricalConcept(
155+
"/foo//bar", "bar", "Bar", "my_dataset", "foo!", List.of("a", "b"), true, "", List.of(), Map.of("key", "value")
156+
);
157+
Concept fooBaz = new ContinuousConcept("/foo//baz", "baz", "Baz", "my_dataset", "foo!", true, 0F, 100F, "", Map.of("key", "value"));
158+
List<Concept> concepts = List.of(fooBar, fooBaz);
159+
List<String> conceptPaths = List.of("/foo//bar", "/foo//bar");
160+
Mockito.when(conceptService.conceptsWithDetail(conceptPaths)).thenReturn(concepts);
161+
ResponseEntity<List<Concept>> listResponseEntity = subject.conceptsDetail(conceptPaths);
162+
Assertions.assertEquals(HttpStatus.OK, listResponseEntity.getStatusCode());
163+
Assertions.assertEquals(concepts, listResponseEntity.getBody());
164+
}
151165
}

‎src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java

+26
Original file line numberDiff line numberDiff line change
@@ -324,5 +324,31 @@ void shouldGetContConceptWithDecimalNotation() {
324324
Assertions.assertEquals(6.77f, concept.max());
325325
}
326326

327+
@Test
328+
void shouldGetConceptsByConceptPath() {
329+
List<String> conceptPaths = List.of(
330+
"\\phs002385\\TXNUM\\", "\\phs000284\\pht001902\\phv00122507\\age\\", "\\phs000007\\pht000022" + "\\phv00004260\\FM219\\",
331+
"\\NHANES\\examination\\physical fitness\\Stage 1 heart rate (per min)", "\\phs000007\\pht000021" + "\\phv00003844\\FL200\\",
332+
"\\phs002715\\age\\"
333+
);
334+
List<Concept> conceptsByPath = subject.getConceptsByPathWithMetadata(conceptPaths);
335+
Assertions.assertFalse(conceptsByPath.isEmpty());
336+
Assertions.assertEquals(6, conceptsByPath.size());
337+
}
338+
339+
@Test
340+
void shouldGetSameConceptMetaAsConceptDetails() {
341+
List<String> conceptPaths = List.of("\\phs002385\\TXNUM\\", "\\phs000284\\pht001902\\phv00122507\\age\\");
342+
List<Concept> conceptsByPath = subject.getConceptsByPathWithMetadata(conceptPaths);
343+
Assertions.assertFalse(conceptsByPath.isEmpty());
344+
345+
// Verify the meta data is correctly retrieve by comparing against known good query.
346+
Concept concept = conceptsByPath.getFirst();
347+
Map<String, String> expectedMeta = subject.getConceptMeta(concept.dataset(), concept.conceptPath());
348+
349+
// compare the maps to each other.
350+
Map<String, String> actualMeta = concept.meta();
351+
Assertions.assertEquals(actualMeta, expectedMeta);
352+
}
327353

328354
}

0 commit comments

Comments
 (0)