[ALS-4947] Add anyRecordOfMulti field to the query object (#77)

Author: ramari16

client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java

@@ -37,6 +37,7 @@ public Query(Query query) {
 	private List<String> fields = new ArrayList<>();
 	private List<String> requiredFields = new ArrayList<>();
 	private List<String> anyRecordOf = new ArrayList<>();
+	private List<List<String>> anyRecordOfMulti = new ArrayList<>();
 	private Map<String, DoubleFilter> numericFilters = new HashMap<>();
 	private Map<String, String[]> categoryFilters = new HashMap<>();
 	private List<VariantInfoFilter> variantInfoFilters = new ArrayList<>();
@@ -62,6 +63,14 @@ public List<String> getRequiredFields() {
 	public List<String> getAnyRecordOf() {
 		return anyRecordOf;
 	}
+	public List<List<String>> getAnyRecordOfMulti() {
+		return anyRecordOfMulti;
+	}
+	public List<List<String>> getAllAnyRecordOf() {
+		List<List<String>> anyRecordOfMultiCopy = new ArrayList<>(anyRecordOfMulti);
+		anyRecordOfMultiCopy.add(anyRecordOf);
+		return anyRecordOfMultiCopy;
+	}
 
 	public Map<String, DoubleFilter> getNumericFilters() {
 		return numericFilters;
@@ -98,6 +107,9 @@ public void setRequiredFields(Collection<String> requiredFields) {
 	public void setAnyRecordOf(Collection<String> anyRecordOf) {
 		this.anyRecordOf = anyRecordOf != null ? new ArrayList<>(anyRecordOf) : new ArrayList<>();
 	}
+	public void setAnyRecordOfMulti(Collection<List<String>> anyRecordOfMulti) {
+		this.anyRecordOfMulti = anyRecordOfMulti != null ? new ArrayList<>(anyRecordOfMulti) : new ArrayList<>();
+	}
 
 	public void setNumericFilters(Map<String, DoubleFilter> numericFilters) {
 		this.numericFilters = numericFilters != null ? new HashMap<>(numericFilters) : new HashMap<>();
@@ -191,7 +203,7 @@ public String toString() {
 		writePartFormat("Numeric filters", numericFilters, builder);
 		writePartFormat("Category filters", categoryFilters, builder);
 		writePartFormat("Variant Info filters", variantInfoFilters, builder, false);
-		writePartFormat("Any-Record-Of filters", anyRecordOf, builder, true);
+		writePartFormat("Any-Record-Of filters", getAllAnyRecordOf(), builder, true);
 
 		return builder.toString();
 	}
@@ -234,7 +246,7 @@ private static void showTopLevelValues(Collection varList, StringBuilder builder
 
 				Integer count = countMap.get(firstLevel);
 				if(count == null) {
-					count = new Integer(1);
+					count = 1;
 				} else {
 					count = count + 1;
 				}

processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessor.java

@@ -6,6 +6,7 @@
 import java.util.Map.Entry;
 import java.util.concurrent.*;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import java.util.zip.GZIPInputStream;
 
 import com.google.common.util.concurrent.UncheckedExecutionException;
@@ -195,24 +196,27 @@ protected Set<Integer> applyBooleanLogic(List<Set<Integer>> filteredIdSets) {
 	 * @return
 	 */
 	protected List<Set<Integer>> idSetsForEachFilter(Query query) {
-		ArrayList<Set<Integer>> filteredIdSets = new ArrayList<Set<Integer>>();
+		final ArrayList<Set<Integer>> filteredIdSets = new ArrayList<>();
 
 		try {
-			addIdSetsForAnyRecordOf(query, filteredIdSets);
+			query.getAllAnyRecordOf().forEach(anyRecordOfFilterList -> {
+				addIdSetsForAnyRecordOf(anyRecordOfFilterList, filteredIdSets);
+			});
 			addIdSetsForRequiredFields(query, filteredIdSets);
 			addIdSetsForNumericFilters(query, filteredIdSets);
 			addIdSetsForCategoryFilters(query, filteredIdSets);
 		} catch (InvalidCacheLoadException e) {
 			log.warn("Invalid query supplied: " + e.getLocalizedMessage());
-			filteredIdSets.add(new HashSet<Integer>()); // if an invalid path is supplied, no patients should match.
+			filteredIdSets.add(new HashSet<>()); // if an invalid path is supplied, no patients should match.
 		}
 
 		//AND logic to make sure all patients match each filter
 		if(filteredIdSets.size()>1) {
-			filteredIdSets = new ArrayList<Set<Integer>>(List.of(applyBooleanLogic(filteredIdSets)));
+			List<Set<Integer>> processedFilteredIdSets = new ArrayList<>(List.of(applyBooleanLogic(filteredIdSets)));
+			return addIdSetsForVariantInfoFilters(query, processedFilteredIdSets);
+		} else {
+			return addIdSetsForVariantInfoFilters(query, filteredIdSets);
 		}
-
-		return addIdSetsForVariantInfoFilters(query, filteredIdSets);
 	}
 
 	/**
@@ -260,22 +264,19 @@ private void addIdSetsForRequiredFields(Query query, ArrayList<Set<Integer>> fil
 		}
 	}
 
-	private void addIdSetsForAnyRecordOf(Query query, ArrayList<Set<Integer>> filteredIdSets) {
-		if(!query.getAnyRecordOf().isEmpty()) {
-			Set<Integer> patientsInScope = new ConcurrentSkipListSet<Integer>();
-			VariantBucketHolder<VariantMasks> bucketCache = new VariantBucketHolder<VariantMasks>();
-			query.getAnyRecordOf().parallelStream().forEach(path->{
-				if(patientsInScope.size()<Math.max(
-						phenotypeMetaStore.getPatientIds().size(),
-						variantService.getPatientIds().length)) {
-					if(VariantUtils.pathIsVariantSpec(path)) {
-						addIdSetsForVariantSpecCategoryFilters(new String[]{"0/1","1/1"}, path, patientsInScope, bucketCache);
-					} else {
-						patientsInScope.addAll(getCube(path).keyBasedIndex());
-					}
+	private void addIdSetsForAnyRecordOf(List<String> anyRecordOfFilters, ArrayList<Set<Integer>> filteredIdSets) {
+		if(!anyRecordOfFilters.isEmpty()) {
+			VariantBucketHolder<VariantMasks> bucketCache = new VariantBucketHolder<>();
+			Set<Integer> anyRecordOfPatientSet = anyRecordOfFilters.parallelStream().flatMap(path -> {
+				if (VariantUtils.pathIsVariantSpec(path)) {
+					TreeSet<Integer> patientsInScope = new TreeSet<>();
+					addIdSetsForVariantSpecCategoryFilters(new String[]{"0/1", "1/1"}, path, patientsInScope, bucketCache);
+					return patientsInScope.stream();
+				} else {
+					return (Stream<Integer>) getCube(path).keyBasedIndex().stream();
 				}
-			});
-			filteredIdSets.add(patientsInScope);
+			}).collect(Collectors.toSet());
+			filteredIdSets.add(anyRecordOfPatientSet);
 		}
 	}
 
@@ -289,9 +290,9 @@ private void addIdSetsForNumericFilters(Query query, ArrayList<Set<Integer>> fil
 
 	private void addIdSetsForCategoryFilters(Query query, ArrayList<Set<Integer>> filteredIdSets) {
 		if(!query.getCategoryFilters().isEmpty()) {
-			VariantBucketHolder<VariantMasks> bucketCache = new VariantBucketHolder<VariantMasks>();
-			Set<Set<Integer>> idsThatMatchFilters = (Set<Set<Integer>>)query.getCategoryFilters().entrySet().parallelStream().map(entry->{
-				Set<Integer> ids = new TreeSet<Integer>();
+			VariantBucketHolder<VariantMasks> bucketCache = new VariantBucketHolder<>();
+			Set<Set<Integer>> idsThatMatchFilters = query.getCategoryFilters().entrySet().parallelStream().map(entry->{
+				Set<Integer> ids = new TreeSet<>();
 				if(VariantUtils.pathIsVariantSpec(entry.getKey())) {
 					addIdSetsForVariantSpecCategoryFilters(entry.getValue(), entry.getKey(), ids, bucketCache);
 				} else {