[ALS-IDK] Stigmatizing variables

Author: Luke Sikina

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java

@@ -5,6 +5,7 @@
 import edu.harvard.dbmi.avillach.dictionary.filter.QueryParamPair;
 import edu.harvard.dbmi.avillach.dictionary.util.MapExtractor;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.domain.Pageable;
 import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
 import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
@@ -17,35 +18,52 @@
 @Repository
 public class ConceptRepository {
 
-    private final NamedParameterJdbcTemplate template;
+    private static final String ALLOW_FILTERING_Q = """
+        WITH allow_filtering AS (
+            SELECT
+                concept_node.concept_node_id AS concept_node_id,
+                (string_agg(concept_node_meta.value, ' ') NOT LIKE '%yes%') AS allowFiltering
+            FROM
+                concept_node
+                JOIN concept_node_meta ON
+                    concept_node.concept_node_id = concept_node_meta.concept_node_id
+                    AND concept_node_meta.KEY IN (:disallowed_meta_keys)
+            GROUP BY
+                concept_node.concept_node_id
+        )
+        """;
 
+    private final NamedParameterJdbcTemplate template;
     private final ConceptRowMapper mapper;
-
     private final ConceptFilterQueryGenerator filterGen;
     private final ConceptMetaExtractor conceptMetaExtractor;
     private final ConceptResultSetExtractor conceptResultSetExtractor;
+    private final List<String> disallowedMetaFields;
 
 
     @Autowired
     public ConceptRepository(
         NamedParameterJdbcTemplate template, ConceptRowMapper mapper, ConceptFilterQueryGenerator filterGen,
-        ConceptMetaExtractor conceptMetaExtractor, ConceptResultSetExtractor conceptResultSetExtractor
+        ConceptMetaExtractor conceptMetaExtractor, ConceptResultSetExtractor conceptResultSetExtractor,
+        @Value("${filtering.unfilterable_concepts}") List<String> disallowedMetaFields
     ) {
         this.template = template;
         this.mapper = mapper;
         this.filterGen = filterGen;
         this.conceptMetaExtractor = conceptMetaExtractor;
         this.conceptResultSetExtractor = conceptResultSetExtractor;
+        this.disallowedMetaFields = disallowedMetaFields;
     }
 
 
     public List<Concept> getConcepts(Filter filter, Pageable pageable) {
-        String sql = """
+        String sql = ALLOW_FILTERING_Q + """
             SELECT
                 concept_node.*,
                 ds.REF as dataset,
                 continuous_min.VALUE as min, continuous_max.VALUE as max,
                 categorical_values.VALUE as values,
+                allow_filtering.allowFiltering AS allowFiltering,
                 meta_description.VALUE AS description
             FROM
                 concept_node
@@ -54,12 +72,13 @@ public List<Concept> getConcepts(Filter filter, Pageable pageable) {
                 LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min'
                 LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max'
                 LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values'
+                LEFT JOIN allow_filtering ON concept_node.concept_node_id = allow_filtering.concept_node_id
             WHERE concept_node.concept_node_id IN (
             
             """;
         QueryParamPair filterQ = filterGen.generateFilterQuery(filter, pageable);
         sql = sql + filterQ.query() + "\n)";
-        MapSqlParameterSource params = filterQ.params();
+        MapSqlParameterSource params = filterQ.params().addValue("disallowed_meta_keys", disallowedMetaFields);
 
         return template.query(sql, params, mapper);
     }
@@ -72,12 +91,13 @@ public long countConcepts(Filter filter) {
     }
 
     public Optional<Concept> getConcept(String dataset, String conceptPath) {
-        String sql = """
+        String sql = ALLOW_FILTERING_Q + """
             SELECT
                 concept_node.*,
                 ds.REF as dataset,
                 continuous_min.VALUE as min, continuous_max.VALUE as max,
                 categorical_values.VALUE as values,
+                allow_filtering.allowFiltering AS allowFiltering,
                 meta_description.VALUE AS description
             FROM
                 concept_node
@@ -86,13 +106,15 @@ public Optional<Concept> getConcept(String dataset, String conceptPath) {
                 LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min'
                 LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max'
                 LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values'
+                LEFT JOIN allow_filtering ON concept_node.concept_node_id = allow_filtering.concept_node_id
             WHERE
                 concept_node.concept_path = :conceptPath
                 AND ds.REF = :dataset
             """;
         MapSqlParameterSource params = new MapSqlParameterSource()
             .addValue("conceptPath", conceptPath)
-            .addValue("dataset", dataset);
+            .addValue("dataset", dataset)
+            .addValue("disallowed_meta_keys", disallowedMetaFields);
         return template.query(sql, params, mapper).stream().findFirst();
     }
 
@@ -137,8 +159,8 @@ public Map<Concept, Map<String, String>> getConceptMetaForConcepts(List<Concept>
     }
 
     public Optional<Concept> getConceptTree(String dataset, String conceptPath, int depth) {
-        String sql = """
-                WITH core_query AS (
+        String sql = ALLOW_FILTERING_Q + """
+                , core_query AS (
                     WITH RECURSIVE nodes AS (
                         SELECT
                             concept_node_id, parent_id, 0 AS depth
@@ -196,6 +218,7 @@ WITH RECURSIVE nodes AS (
                     continuous_min.VALUE AS min, continuous_max.VALUE AS max,
                     categorical_values.VALUE AS values,
                     meta_description.VALUE AS description,
+                    allow_filtering.allowFiltering AS allowFiltering,
                     core_query.depth AS depth
                 FROM
                     concept_node
@@ -205,11 +228,13 @@ WITH RECURSIVE nodes AS (
                     LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min'
                     LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max'
                     LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values'
+                    LEFT JOIN allow_filtering ON concept_node.concept_node_id = allow_filtering.concept_node_id
             """;
         MapSqlParameterSource params = new MapSqlParameterSource()
             .addValue("path", conceptPath)
             .addValue("dataset", dataset)
-            .addValue("depth", depth);
+            .addValue("depth", depth)
+            .addValue("disallowed_meta_keys", disallowedMetaFields);
 
         if (depth < 0) {
             return Optional.empty();

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java

@@ -20,6 +20,7 @@ public CategoricalConcept mapCategorical(ResultSet rs) throws SQLException {
             rs.getString("concept_path"), rs.getString("name"),
             rs.getString("display"), rs.getString("dataset"), rs.getString("description"),
             rs.getString("values") == null ? List.of() : parseValues(rs.getString("values")),
+            rs.getBoolean("allowFiltering"),
             null,
             null
         );
@@ -29,6 +30,7 @@ public ContinuousConcept mapContinuous(ResultSet rs) throws SQLException {
         return new ContinuousConcept(
             rs.getString("concept_path"), rs.getString("name"),
             rs.getString("display"), rs.getString("dataset"), rs.getString("description"),
+            rs.getBoolean("allowFiltering"),
             parseMin(rs.getString("values")), parseMax(rs.getString("values")),
             null
         );

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/CategoricalConcept.java

@@ -10,7 +10,7 @@
 public record CategoricalConcept(
     String conceptPath, String name, String display, String dataset, String description,
 
-    List<String> values,
+    List<String> values, boolean allowFiltering,
 
     @Nullable
     List<Concept> children,
@@ -28,21 +28,17 @@ public record CategoricalConcept(
 
     public CategoricalConcept(
         String conceptPath, String name, String display, String dataset, String description, List<String> values,
-        @Nullable List<Concept> children, @Nullable Map<String, String> meta
+        boolean allowFiltering, @Nullable List<Concept> children, @Nullable Map<String, String> meta
     ) {
-        this(conceptPath, name, display, dataset, description, values, children, meta, null, null);
+        this(conceptPath, name, display, dataset, description, values, allowFiltering, children, meta, null, null);
     }
 
     public CategoricalConcept(CategoricalConcept core, Map<String, String> meta) {
-        this(core.conceptPath, core.name, core.display, core.dataset, core.description, core.values, core.children, meta);
-    }
-
-    public CategoricalConcept(CategoricalConcept core, List<Concept> children) {
-        this(core.conceptPath, core.name, core.display, core.dataset, core.description, core.values, children, core.meta);
+        this(core.conceptPath, core.name, core.display, core.dataset, core.description, core.values, core.allowFiltering, core.children, meta);
     }
 
     public CategoricalConcept(String conceptPath, String dataset) {
-        this(conceptPath, "", "", dataset, "", List.of(), List.of(), null);
+        this(conceptPath, "", "", dataset, "", List.of(), false, List.of(), null);
     }
 
 
@@ -54,20 +50,20 @@ public ConceptType type() {
 
     @Override
     public CategoricalConcept withChildren(List<Concept> children) {
-        return new CategoricalConcept(this, children);
+        return new CategoricalConcept(conceptPath, name, display, dataset, description, values, allowFiltering, children, meta);
     }
 
     @Override
     public Concept withTable(Concept table) {
         return new CategoricalConcept(
-            conceptPath, name, display, dataset, description, values, children, meta, table, study
+            conceptPath, name, display, dataset, description, values, allowFiltering, children, meta, table, study
         );
     }
 
     @Override
     public Concept withStudy(Concept study) {
         return new CategoricalConcept(
-            conceptPath, name, display, dataset, description, values, children, meta, table, study
+            conceptPath, name, display, dataset, description, values, allowFiltering, children, meta, table, study
         );
     }
 

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/Concept.java

@@ -58,6 +58,10 @@ public sealed interface Concept
     @Nullable
     List<Concept> children();
 
+    default boolean allowFiltering() {
+        return false;
+    }
+
     Concept withChildren(List<Concept> children);
 
     Concept withTable(Concept table);

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java

@@ -9,7 +9,7 @@
 import java.util.Objects;
 
 public record ContinuousConcept(
-    String conceptPath, String name, String display, String dataset, String description,
+    String conceptPath, String name, String display, String dataset, String description, boolean allowFiltering,
 
     @Nullable Integer min, @Nullable Integer max,
     Map<String, String> meta,
@@ -24,29 +24,25 @@ public record ContinuousConcept(
 ) implements Concept {
 
     public ContinuousConcept(
-        String conceptPath, String name, String display, String dataset, String description,
+        String conceptPath, String name, String display, String dataset, String description, boolean allowFiltering,
         @Nullable Integer min, @Nullable Integer max, Map<String, String> meta, @Nullable List<Concept> children
     ) {
-        this(conceptPath, name, display, dataset, description, min, max, meta, children, null, null);
+        this(conceptPath, name, display, dataset, description, allowFiltering, min, max, meta, children, null, null);
     }
 
     public ContinuousConcept(ContinuousConcept core, Map<String, String> meta) {
-        this(core.conceptPath, core.name, core.display, core.dataset, core.description, core.min, core.max, meta, core.children);
-    }
-
-    public ContinuousConcept(ContinuousConcept core, List<Concept> children) {
-        this(core.conceptPath, core.name, core.display, core.dataset, core.description, core.min, core.max, core.meta, children);
+        this(core.conceptPath, core.name, core.display, core.dataset, core.description, core.allowFiltering, core.min, core.max, meta, core.children);
     }
 
     public ContinuousConcept(String conceptPath, String dataset) {
-        this(conceptPath, "", "", dataset, "", null, null, null, List.of());
+        this(conceptPath, "", "", dataset, "", true, null, null, null, List.of());
     }
 
     public ContinuousConcept(
-        String conceptPath, String name, String display, String dataset, String description,
+        String conceptPath, String name, String display, String dataset, String description, boolean allowFiltering,
         @Nullable Integer min, @Nullable Integer max, Map<String, String> meta
     ) {
-        this(conceptPath, name, display, dataset, description, min, max, meta, null);
+        this(conceptPath, name, display, dataset, description, allowFiltering, min, max, meta, null);
     }
 
     @JsonProperty("type")
@@ -57,20 +53,20 @@ public ConceptType type() {
 
     @Override
     public ContinuousConcept withChildren(List<Concept> children) {
-        return new ContinuousConcept(this, children);
+        return new ContinuousConcept(conceptPath, name, display, dataset, description, allowFiltering, min, max, meta, children);
     }
 
     @Override
     public Concept withTable(Concept table) {
         return new ContinuousConcept(
-            conceptPath, name, display, dataset, description, min, max, meta, children, table, study
+            conceptPath, name, display, dataset, description, allowFiltering, min, max, meta, children, table, study
         );
     }
 
     @Override
     public Concept withStudy(Concept study) {
         return new ContinuousConcept(
-            conceptPath, name, display, dataset, description, min, max, meta, children, table, study
+            conceptPath, name, display, dataset, description, allowFiltering, min, max, meta, children, table, study
         );
     }
 

src/main/resources/application-bdc.properties

@@ -5,4 +5,6 @@ spring.datasource.url=jdbc-secretsmanager:postgresql://${DATASOURCE_URL}/picsure
 spring.datasource.username=${DATASOURCE_USERNAME}
 server.port=80
 
-dashboard.enable.extra_details=true
+dashboard.enable.extra_details=true
+
+filtering.unfilterable_concepts=stigmatizing

src/main/resources/application.properties

@@ -8,4 +8,6 @@ server.port=80
 dashboard.columns={abbreviation:'Abbreviation',name:'Name',clinvars:'Clinical Variables'}
 dashboard.column-order=abbreviation,name,clinvars
 dashboard.nonmeta-columns=abbreviation,name
-dashboard.enable.extra_details=true
+dashboard.enable.extra_details=true
+
+filtering.unfilterable_concepts=stigmatizing