TASK-7134 - Re-implement Aggregations Stats for all Catalog Browsers

commons-datastore/commons-datastore-core/src/main/java/org/opencb/commons/datastore/core/FacetField.java

@@ -21,10 +21,9 @@
 /**
  * Created by jtarraga on 09/03/17.
  */
-
 public class FacetField {
     private String name;
-    private long count;
+    private Long count;
     private List<Bucket> buckets;
     private String aggregationName;
     private List<Double> aggregationValues;
@@ -38,6 +37,13 @@ public FacetField(String name, long count, List<Bucket> buckets) {
         this.buckets = buckets;
     }
 
+    public FacetField(String name, long count, String aggregationName, List<Double> aggregationValues) {
+        this.name = name;
+        this.count = count;
+        this.aggregationName = aggregationName;
+        this.aggregationValues = aggregationValues;
+    }
+
     public FacetField(String name, String aggregationName, List<Double> aggregationValues) {
         this.name = name;
         this.aggregationName = aggregationName;
@@ -68,17 +74,20 @@ public FacetField setName(String name) {
         return this;
     }
 
-    public long getCount() {
+    public Long getCount() {
         return count;
     }
 
-    public FacetField setCount(long count) {
+    public FacetField setCount(Long count) {
         this.count = count;
         return this;
     }
 
     public FacetField addCount(long delta) {
-        this.count += delta;
+        if (this.count == null) {
+            this.count = 0L;
+        }
+        this.count = this.count.longValue() + delta;
         return this;
     }
 

commons-datastore/commons-datastore-mongodb/src/main/java/org/opencb/commons/datastore/mongodb/MongoDBCollection.java

@@ -97,8 +97,7 @@ private <T> DataResult<T> endQuery(List result, long numMatches, double start) {
         long end = System.currentTimeMillis();
         int numResults = (result != null) ? result.size() : 0;
 
-        DataResult<T> queryResult = new DataResult((int) (end - start), Collections.emptyList(), numResults, result, numMatches, null);
-        return queryResult;
+        return new DataResult((int) (end - start), Collections.emptyList(), numResults, result, numMatches, null);
     }
 
     private DataResult endWrite(long start) {
@@ -331,31 +330,25 @@ public <T> DataResult<T> aggregate(List<? extends Bson> operations, ComplexTypeC
                                        QueryOptions options) {
 
         long start = startQuery();
-
         DataResult<T> queryResult;
-        MongoDBIterator<T> iterator = mongoDBNativeQuery.aggregate(operations, converter, options);
-//        MongoCursor<Document> iterator = output.iterator();
         List<T> list = new LinkedList<>();
-        if (queryResultWriter != null) {
-            try {
-                queryResultWriter.open();
+        if (operations != null && !operations.isEmpty()) {
+            MongoDBIterator<T> iterator = mongoDBNativeQuery.aggregate(operations, converter, options);
+            if (queryResultWriter != null) {
+                try {
+                    queryResultWriter.open();
+                    while (iterator.hasNext()) {
+                        queryResultWriter.write(iterator.next());
+                    }
+                    queryResultWriter.close();
+                } catch (IOException e) {
+                    throw new RuntimeException(e.getMessage(), e);
+                }
+            } else {
                 while (iterator.hasNext()) {
-                    queryResultWriter.write(iterator.next());
+                    list.add(iterator.next());
                 }
-                queryResultWriter.close();
-            } catch (IOException e) {
-                throw new RuntimeException(e.getMessage(), e);
-            }
-        } else {
-//            if (converter != null) {
-//                while (iterator.hasNext()) {
-//                    list.add(converter.convertToDataModelType(iterator.next()));
-//                }
-//            } else {
-            while (iterator.hasNext()) {
-                list.add((T) iterator.next());
             }
-//            }
         }
         queryResult = endQuery(list, start);
         return queryResult;
@@ -435,7 +428,7 @@ public DataResult update(ClientSession clientSession, List<? extends Bson> queri
 
         return endWrite(
                 wr.getMatchedCount(),
-                wr.getInsertedCount() + wr.getUpserts().size(),
+                (long) wr.getInsertedCount() + wr.getUpserts().size(),
                 wr.getModifiedCount(),
                 wr.getDeletedCount(),
                 0,
@@ -553,8 +546,7 @@ public DataResult createIndex(Bson keys, ObjectMap options) {
         }
 
         mongoDBNativeQuery.createIndex(keys, i);
-        DataResult dataResult = endQuery(Collections.emptyList(), start);
-        return dataResult;
+        return endQuery(Collections.emptyList(), start);
     }
 
     public void dropIndexes() {
@@ -564,15 +556,13 @@ public void dropIndexes() {
     public DataResult dropIndex(Bson keys) {
         long start = startQuery();
         mongoDBNativeQuery.dropIndex(keys);
-        DataResult dataResult = endQuery(Collections.emptyList(), start);
-        return dataResult;
+        return endQuery(Collections.emptyList(), start);
     }
 
     public DataResult<Document> getIndex() {
         long start = startQuery();
         List<Document> index = mongoDBNativeQuery.getIndex();
-        DataResult<Document> queryResult = endQuery(index, start);
-        return queryResult;
+        return endQuery(index, start);
     }
 
 

commons-datastore/commons-datastore-mongodb/src/main/java/org/opencb/commons/datastore/mongodb/MongoDBDocumentToFacetFieldsConverter.java

@@ -0,0 +1,191 @@
+package org.opencb.commons.datastore.mongodb;
+
+import org.apache.commons.lang3.StringUtils;
+import org.bson.Document;
+import org.opencb.commons.datastore.core.ComplexTypeConverter;
+import org.opencb.commons.datastore.core.FacetField;
+
+import java.util.*;
+
+import static org.opencb.commons.datastore.mongodb.GenericDocumentComplexConverter.TO_REPLACE_DOTS;
+import static org.opencb.commons.datastore.mongodb.MongoDBQueryUtils.*;
+import static org.opencb.commons.datastore.mongodb.MongoDBQueryUtils.Accumulator.*;
+
+public class MongoDBDocumentToFacetFieldsConverter implements ComplexTypeConverter<List<FacetField>, Document> {
+
+    @Override
+    public List<FacetField> convertToDataModelType(Document document) {
+        if (document == null || document.entrySet().size() == 0) {
+            return Collections.emptyList();
+        }
+
+        String facetFieldName;
+        List<FacetField> facets = new ArrayList<>();
+        for (Map.Entry<String, Object> entry : document.entrySet()) {
+            String key = entry.getKey();
+            List<Document> documentValues = (List<Document>) entry.getValue();
+            if (key.endsWith(COUNTS_SUFFIX) || key.endsWith(FACET_ACC_SUFFIX)) {
+                List<FacetField.Bucket> buckets = new ArrayList<>(documentValues.size());
+                long total = 0;
+                for (Document documentValue : documentValues) {
+                    long counter = documentValue.getInteger(count.name());
+                    String bucketValue = "";
+                    Object internalIdValue = documentValue.get(INTERNAL_ID);
+                    if (internalIdValue instanceof String) {
+                        bucketValue = (String) internalIdValue;
+                    } else if (internalIdValue instanceof Boolean
+                            || internalIdValue instanceof Integer
+                            || internalIdValue instanceof Long
+                            || internalIdValue instanceof Double) {
+                        bucketValue = internalIdValue.toString();
+                    } else if (internalIdValue instanceof Document) {
+                        bucketValue = StringUtils.join(((Document) internalIdValue).values(), AND_SEPARATOR);
+                    }
+
+                    List<FacetField> bucketFacetFields = null;
+                    if (key.endsWith(FACET_ACC_SUFFIX)) {
+                        String[] split = key.split(SEPARATOR);
+                        String name = split[2];
+                        String aggregationName = split[1];
+                        Double value;
+                        if (documentValue.get(aggregationName) instanceof Integer) {
+                            value = 1.0d * documentValue.getInteger(aggregationName);
+                        } else if (documentValue.get(aggregationName) instanceof Long) {
+                            value = 1.0d * documentValue.getLong(aggregationName);
+                        } else {
+                            value = documentValue.getDouble(aggregationName);
+                        }
+                        List<Double> aggregationValues = Collections.singletonList(value);
+                        FacetField facetField = new FacetField(name.replace(TO_REPLACE_DOTS, "."), aggregationName, aggregationValues);
+                        // Perhaps it’s redundant, as it is also set in the bucket
+                        facetField.setCount(counter);
+                        bucketFacetFields = Collections.singletonList(facetField);
+                    }
+
+                    buckets.add(new FacetField.Bucket(bucketValue, counter, bucketFacetFields));
+                    total += counter;
+                }
+                facetFieldName = key.split(SEPARATOR)[0].replace(TO_REPLACE_DOTS, ".");
+                facets.add(new FacetField(facetFieldName, total, buckets));
+            } else if (key.endsWith(RANGES_SUFFIX)) {
+                List<Double> facetFieldValues = new ArrayList<>();
+                Number start = null;
+                Number end = null;
+                Number step = null;
+                Double other = null;
+                for (Document value : documentValues) {
+                    if (value.get(INTERNAL_ID) instanceof String && OTHER.equals(value.getString(INTERNAL_ID))) {
+                        other = 1.0d * value.getInteger(count.name());
+                    } else {
+                        Double range = value.getDouble(INTERNAL_ID);
+                        Integer counter = value.getInteger(count.name());
+                        facetFieldValues.add(1.0d * counter);
+                        if (start == null) {
+                            start = range;
+                        }
+                        end = range;
+                        if (step == null && start != end) {
+                            step = end.doubleValue() - start.doubleValue();
+                        }
+                    }
+                }
+                facetFieldName = key.split(SEPARATOR)[0].replace(TO_REPLACE_DOTS, ".");
+                if (other != null) {
+                    facetFieldName += " (counts out of range: " + other + ")";
+                }
+                FacetField facetField = new FacetField(facetFieldName, "range", facetFieldValues)
+                        .setStart(start)
+                        .setEnd(end)
+                        .setStep(step);
+                facets.add(facetField);
+            } else {
+                if (key.endsWith(RANGES_SUFFIX)) {
+                    List<Double> facetFieldValues = new ArrayList<>();
+                    Number start = null;
+                    Number end = null;
+                    Number step = null;
+                    Double other = null;
+                    for (Document value : documentValues) {
+                        if (value.get(INTERNAL_ID) instanceof String && OTHER.equals(value.getString(INTERNAL_ID))) {
+                            other = 1.0d * value.getInteger(count.name());
+                        } else {
+                            Double range = value.getDouble(INTERNAL_ID);
+                            Integer counter = value.getInteger(count.name());
+                            facetFieldValues.add(1.0d * counter);
+                            if (start == null) {
+                                start = range;
+                            }
+                            end = range;
+                            if (step == null && start != end) {
+                                step = end.doubleValue() - start.doubleValue();
+                            }
+                        }
+                    }
+                    facetFieldName = key.substring(0, key.length() - RANGES_SUFFIX.length()).replace(TO_REPLACE_DOTS, ".");
+                    if (other != null) {
+                        facetFieldName += " (counts out of range: " + other + ")";
+                    }
+                    FacetField facetField = new FacetField(facetFieldName, "range", facetFieldValues)
+                            .setStart(start)
+                            .setEnd(end)
+                            .setStep(step);
+                    facets.add(facetField);
+                } else {
+                    Document documentValue = ((List<Document>) entry.getValue()).get(0);
+                    MongoDBQueryUtils.Accumulator accumulator = getAccumulator(documentValue);
+                    switch (accumulator) {
+                        case sum:
+                        case avg:
+                        case max:
+                        case min:
+                        case stdDevPop:
+                        case stdDevSamp: {
+                            List<Double> fieldValues = new ArrayList<>();
+                            if (documentValue.get(accumulator.name()) instanceof Integer) {
+                                fieldValues.add(1.0d * documentValue.getInteger(accumulator.name()));
+                            } else if (documentValue.get(accumulator.name()) instanceof Long) {
+                                fieldValues.add(1.0d * documentValue.getLong(accumulator.name()));
+                            } else if (documentValue.get(accumulator.name()) instanceof List) {
+                                List<Number> list = (List<Number>) documentValue.get(accumulator.name());
+                                for (Number number : list) {
+                                    fieldValues.add(number.doubleValue());
+                                }
+                            } else {
+                                fieldValues.add(documentValue.getDouble(accumulator.name()));
+                            }
+                            long count = 0;
+                            if (documentValue.containsKey("count")) {
+                                count = Long.valueOf(documentValue.getInteger("count"));
+                            }
+                            facetFieldName = documentValue.getString(INTERNAL_ID).replace(TO_REPLACE_DOTS, ".");
+                            facets.add(new FacetField(facetFieldName, count, accumulator.name(), fieldValues));
+                            break;
+                        }
+                        default: {
+                            // Do nothing, exception is raised
+                        }
+                    }
+                }
+            }
+        }
+        return facets;
+    }
+
+    private MongoDBQueryUtils.Accumulator getAccumulator(Document document) {
+        for (Map.Entry<String, Object> entry : document.entrySet()) {
+            try {
+                MongoDBQueryUtils.Accumulator accumulator = MongoDBQueryUtils.Accumulator.valueOf(entry.getKey());
+                return accumulator;
+            } catch (IllegalArgumentException e) {
+                // Do nothing
+            }
+        }
+        throw new IllegalArgumentException("No accumulators found in facet document: " + StringUtils.join(document.keySet(), ", ")
+                + "Valid accumulator functions: " + StringUtils.join(Arrays.asList(count, sum, max, min, avg, stdDevPop, stdDevSamp), ","));
+    }
+
+    @Override
+    public Document convertToStorageType(List<FacetField> facetFields) {
+        throw new RuntimeException("Not yet implemented");
+    }
+}