extract custom data samples without fetching all samples and modify s…

…ql to account for NAs
gblaih · Jul 29, 2024 · eae97c9 · eae97c9
1 parent 0f95528
commit eae97c9
Show file tree

Hide file tree

Showing 3 changed files with 90 additions and 17 deletions.
diff --git a/src/main/java/org/cbioportal/web/columnar/util/CustomDataFilterUtil.java b/src/main/java/org/cbioportal/web/columnar/util/CustomDataFilterUtil.java
@@ -52,18 +52,7 @@ public List<SampleIdentifier> extractCustomDataSamples(final StudyViewFilter stu
 
         List<SampleIdentifier> sampleIdentifiers = new ArrayList<>();
 
-        if (studyViewFilter != null && studyViewFilter.getSampleIdentifiers() != null && !studyViewFilter.getSampleIdentifiers().isEmpty()) {
-            List<String> studyIds = new ArrayList<>();
-            List<String> sampleIds = new ArrayList<>();
-            studyViewFilterUtil.extractStudyAndSampleIds(studyViewFilter.getSampleIdentifiers(), studyIds, sampleIds);
-            sampleIdentifiers = sampleService.fetchSamples(studyIds, sampleIds, Projection.ID.name()).stream()
-                .map(sampleToSampleIdentifier).collect(Collectors.toList());
-        } else {
-            sampleIdentifiers = sampleService.getAllSamplesInStudies(studyViewFilter.getStudyIds(), Projection.ID.name(),
-                null, null, null, null).stream().map(sampleToSampleIdentifier).collect(Collectors.toList());
-        }
-
-        if (CollectionUtils.isEmpty(studyViewFilter.getCustomDataFilters()) || sampleIdentifiers.isEmpty()) {
+        if (CollectionUtils.isEmpty(studyViewFilter.getCustomDataFilters())) {
             return null;
         }
 
@@ -91,6 +80,10 @@ public List<SampleIdentifier> extractCustomDataSamples(final StudyViewFilter stu
                 if (value.equals("NAN") || value.equals("N/A")) {
                     value = "NA";
                 }
+                SampleIdentifier sampleIdentifier =
+                    studyViewFilterUtil.buildSampleIdentifier(datum.getStudyId(), datum.getSampleId());
+                sampleIdentifier.setAttributeId(customDataSession.getId());
+                sampleIdentifiers.add(sampleIdentifier);
                 customDataByStudySampleSession.put(datum.getStudyId(), datum.getSampleId(), customDataSession.getId(), value);
             })
         );
@@ -115,8 +108,24 @@ public List<SampleIdentifier> extractCustomDataSamples(final StudyViewFilter stu
             }
         });
 
+        List<String> filteredValues = new ArrayList<>();
+        List<String> attributeIdsWithNA = new ArrayList<>();
+
+        for (ClinicalDataFilter s : equalityFilters) {
+            filteredValues = s.getValues()
+                .stream()
+                .map(DataFilterValue::getValue)
+                .collect(Collectors.toList());
+            System.out.println(filteredValues);
+            if (filteredValues.contains("NA")) {
+                attributeIdsWithNA.add(s.getAttributeId());
+            }
+        }
+
         List<SampleIdentifier> filtered = new ArrayList<>();
         sampleIdentifiers.forEach(sampleIdentifier -> {
+            sampleIdentifier.setAllAttributeIds(attributeIds);
+            sampleIdentifier.setAttributeIdsWithNA(attributeIdsWithNA);
             int equalityFilterCount = studyViewFilterUtil.getFilteredCountByDataEquality(equalityFilters, customDataByStudySampleSession,
                 sampleIdentifier.getSampleId(), sampleIdentifier.getStudyId(), false);
             int intervalFilterCount = getFilteredCountByDataInterval(intervalFilters, customDataByStudySampleSession,
@@ -126,6 +135,10 @@ public List<SampleIdentifier> extractCustomDataSamples(final StudyViewFilter stu
             ) {
                 filtered.add(sampleIdentifier);
             }
+            else {
+                sampleIdentifier.setIsFilteredOut(true);
+                filtered.add(sampleIdentifier);
+            }
         });
 
         return filtered;

diff --git a/src/main/java/org/cbioportal/web/parameter/SampleIdentifier.java b/src/main/java/org/cbioportal/web/parameter/SampleIdentifier.java
@@ -1,12 +1,18 @@
 package org.cbioportal.web.parameter;
 
 import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Objects;
 
 public class SampleIdentifier implements Serializable {
 
     private String sampleId;
     private String studyId;
+    private boolean isFilteredOut = false;
+    private List<String> allAttributeIds;
+    private String attributeId;
+    private List<String> attributeIdsWithNA = new ArrayList<>();
 
     public String getSampleId() {
         return sampleId;
@@ -24,6 +30,38 @@ public void setStudyId(String studyId) {
         this.studyId = studyId;
     }
 
+    public boolean getIsFilteredOut() {
+        return isFilteredOut;
+    }
+
+    public void setIsFilteredOut(boolean isFilteredOut) {
+        this.isFilteredOut = isFilteredOut;
+    }
+
+    public List<String> getAllAttributeIds() {
+        return allAttributeIds;
+    }
+
+    public void setAllAttributeIds(List<String> allAttributeIds) {
+        this.allAttributeIds = allAttributeIds;
+    }
+
+    public String getAttributeId() {
+        return attributeId;
+    }
+
+    public void setAttributeId(String attributeId) {
+        this.attributeId = attributeId;
+    }
+
+    public List<String> getAttributeIdsWithNA() {
+        return attributeIdsWithNA;
+    }
+
+    public void setAttributeIdsWithNA(List<String> attributeIds) {
+        this.attributeIdsWithNA = attributeIds;
+    }
+
     @Override
     public String toString() {
         return getSampleId() + getStudyId();

diff --git a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml
@@ -71,20 +71,42 @@
                 INTERSECT
                 SELECT sample_unique_id
                 FROM sample_derived
-                WHERE sample_unique_id IN (
                 <!-- customDataSamples are samples filtered based on the customDataFilters -->
                 <!-- filter for samples in customDataSamples if customDataSamples isn't empty, else all samples are filtered out -->
                 <choose>
                     <when test="!customDataSamples.isEmpty()">
-                        <foreach item="sampleIdentifier" collection="customDataSamples" separator=",">
-                            '${sampleIdentifier.studyId}_${sampleIdentifier.sampleId}'
-                        </foreach>
+                        WHERE
+                        <trim prefix="" prefixOverrides="AND">
+                            <foreach item="attributeId" collection="customDataSamples.get(0).getAllAttributeIds()">
+                                AND
+                                (
+                                    sample_unique_id IN (
+                                        '',
+                                        <foreach item="sampleIdentifier" collection="customDataSamples" separator=",">
+                                            <if test="attributeId == sampleIdentifier.getAttributeId() and !sampleIdentifier.getIsFilteredOut()">
+                                                '${sampleIdentifier.studyId}_${sampleIdentifier.sampleId}'
+                                            </if>
+                                        </foreach>
+                                    )
+                                    <if test="customDataSamples.get(0).getAttributeIdsWithNA().contains(attributeId)">
+                                    OR
+                                    sample_unique_id NOT IN (
+                                        '',
+                                        <foreach item="sampleIdentifier" collection="customDataSamples" separator=",">
+                                            <if test="attributeId == sampleIdentifier.getAttributeId()">
+                                                '${sampleIdentifier.studyId}_${sampleIdentifier.sampleId}'
+                                            </if>
+                                        </foreach>
+                                    )
+                                    </if>
+                                )
+                            </foreach>
+                        </trim>
                     </when>
                     <otherwise>
                         ''
                     </otherwise>
                 </choose>
-                )
             </if>
             <if test="studyViewFilter.geneFilters != null and !studyViewFilter.geneFilters.isEmpty()">
                 <foreach item="profileGroup" collection="studyViewFilter.geneFilters">