DHIS-16705 Convert enrollment analytics sql to use CTE instead of sub…

…queries

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEUtils.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2004-2024, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.hisp.dhis.analytics.common;
+
+import lombok.experimental.UtilityClass;
+import org.hisp.dhis.common.QueryItem;
+
+@UtilityClass
+public class CTEUtils {
+
+  public static String computeKey(QueryItem queryItem) {
+
+    if (queryItem.hasProgramStage()) {
+      return "%s_%s".formatted(queryItem.getProgramStage().getUid(), queryItem.getItemId());
+    } else if (queryItem.isProgramIndicator()) {
+      return queryItem.getItemId();
+    }
+
+    // TODO continue with the rest of the method
+    return "";
+  }
+
+  public static String getIdentifier(QueryItem queryItem) {
+    String stage = queryItem.hasProgramStage() ? queryItem.getProgramStage().getUid() : "default";
+    return stage + "." + queryItem.getItemId();
+  }
+}

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CteContext.java

@@ -0,0 +1,137 @@
+/*
+ * Copyright (c) 2004-2024, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.hisp.dhis.analytics.common;
+
+import static org.hisp.dhis.analytics.common.CTEUtils.computeKey;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+import org.hisp.dhis.common.QueryItem;
+import org.hisp.dhis.program.ProgramIndicator;
+import org.hisp.dhis.program.ProgramStage;
+
+public class CteContext {
+  private final Map<String, CteDefinition> cteDefinitions = new LinkedHashMap<>();
+
+  public CteDefinition getDefinitionByItemUid(String itemUid) {
+    return cteDefinitions.get(itemUid);
+  }
+
+  /**
+   * Adds a CTE definition to the context.
+   *
+   * @param programStage The program stage
+   * @param item The query item
+   * @param cteDefinition The CTE definition (the SQL query)
+   * @param offset The calculated offset
+   * @param isRowContext Whether the CTE is a row context
+   */
+  public void addCte(
+      ProgramStage programStage,
+      QueryItem item,
+      String cteDefinition,
+      int offset,
+      boolean isRowContext) {
+    String key = computeKey(item);
+    if (cteDefinitions.containsKey(key)) {
+      cteDefinitions.get(key).getOffsets().add(offset);
+    } else {
+      var cteDef =
+          new CteDefinition(
+              programStage.getUid(), item.getItemId(), cteDefinition, offset, isRowContext);
+      cteDefinitions.put(key, cteDef);
+    }
+  }
+
+  public void addExistsCte(ProgramStage programStage, QueryItem item, String cteDefinition) {
+    var cteDef =
+        new CteDefinition(programStage.getUid(), item.getItemId(), cteDefinition, -999, false)
+            .setExists(true);
+    cteDefinitions.put(programStage.getUid(), cteDef);
+  }
+
+  /**
+   * Adds a CTE definition to the context.
+   *
+   * @param programIndicator The program indicator
+   * @param cteDefinition The CTE definition (the SQL query)
+   * @param functionRequiresCoalesce Whether the function requires to be "wrapped" in coalesce to
+   *     avoid null values (e.g. avg, sum)
+   */
+  public void addProgramIndicatorCte(
+      ProgramIndicator programIndicator, String cteDefinition, boolean functionRequiresCoalesce) {
+    cteDefinitions.put(
+        programIndicator.getUid(),
+        new CteDefinition(programIndicator.getUid(), cteDefinition, functionRequiresCoalesce));
+  }
+
+  public void addCteFilter(QueryItem item, String ctedefinition) {
+    String key = computeKey(item);
+    if (!cteDefinitions.containsKey(key)) {
+      ProgramStage programStage = item.getProgramStage();
+      cteDefinitions.put(
+          key,
+          new CteDefinition(
+              item.getItemId(),
+              programStage == null ? null : programStage.getUid(),
+              ctedefinition,
+              true));
+    }
+  }
+
+  public String getCteDefinition() {
+    if (cteDefinitions.isEmpty()) {
+      return "";
+    }
+
+    StringBuilder sb = new StringBuilder("with ");
+    boolean first = true;
+    for (Map.Entry<String, CteDefinition> entry : cteDefinitions.entrySet()) {
+      if (!first) {
+        sb.append(", ");
+      }
+      CteDefinition cteDef = entry.getValue();
+      sb.append(cteDef.asCteName(entry.getKey()))
+          .append(" AS (")
+          .append(entry.getValue().getCteDefinition())
+          .append(")");
+      first = false;
+    }
+    return sb.toString();
+  }
+
+  // Rename to item uid
+  public Set<String> getCteNames() {
+    return cteDefinitions.keySet();
+  }
+
+  public boolean containsCte(String cteName) {
+    return cteDefinitions.containsKey(cteName);
+  }
+}

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CteDefinition.java

@@ -0,0 +1,153 @@
+/*
+ * Copyright (c) 2004-2025, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.hisp.dhis.analytics.common;
+
+import java.util.ArrayList;
+import java.util.List;
+import lombok.Getter;
+import org.apache.commons.text.RandomStringGenerator;
+
+public class CteDefinition {
+
+  // Query item id
+  @Getter private String itemId;
+  // The program stage uid
+  private final String programStageUid;
+  // The program indicator uid
+  private String programIndicatorUid;
+  // The CTE definition (the SQL query)
+  @Getter private final String cteDefinition;
+  // The calculated offset
+  @Getter private final List<Integer> offsets = new ArrayList<>();
+  // The alias of the CTE
+  private final String alias;
+  // Whether the CTE is a row context (TODO this need a better explanation)
+  @Getter private boolean rowContext;
+  // Whether the CTE is a program indicator
+  @Getter private boolean programIndicator = false;
+  // Whether the CTE is a filter
+  @Getter private boolean filter = false;
+  // Whether the CTE is a exists, used for checking if the enrollment exists
+  private boolean isExists = false;
+
+  @Getter private boolean requiresCoalesce = false;
+
+  private static final String PS_PREFIX = "ps";
+  private static final String PI_PREFIX = "pi";
+
+  public CteDefinition setExists(boolean exists) {
+    this.isExists = exists;
+    return this;
+  }
+
+  public String getAlias() {
+    if (offsets.isEmpty()) {
+      return alias;
+    }
+    return computeAlias(offsets.get(0));
+  }
+
+  public String getAlias(int offset) {
+    return computeAlias(offset);
+  }
+
+  private String computeAlias(int offset) {
+    return alias + "_" + offset;
+  }
+
+  public CteDefinition(
+      String programStageUid, String queryItemId, String cteDefinition, int offset) {
+    this.programStageUid = programStageUid;
+    this.itemId = queryItemId;
+    this.cteDefinition = cteDefinition;
+    this.offsets.add(offset);
+    // one alias per offset
+    this.alias = new RandomStringGenerator.Builder().withinRange('a', 'z').build().generate(5);
+    this.rowContext = false;
+  }
+
+  public CteDefinition(
+      String programStageUid,
+      String queryItemId,
+      String cteDefinition,
+      int offset,
+      boolean isRowContext) {
+    this(programStageUid, queryItemId, cteDefinition, offset);
+    this.rowContext = isRowContext;
+  }
+
+  public CteDefinition(String programIndicatorUid, String cteDefinition, boolean requiresCoalesce) {
+    this.cteDefinition = cteDefinition;
+    this.programIndicatorUid = programIndicatorUid;
+    this.programStageUid = null;
+    // ignore offset
+    this.alias = new RandomStringGenerator.Builder().withinRange('a', 'z').build().generate(5);
+    this.rowContext = false;
+    this.programIndicator = true;
+    this.requiresCoalesce = requiresCoalesce;
+  }
+
+  public CteDefinition(
+      String queryItemId, String programStageUid, String cteDefinition, boolean isFilter) {
+    this.itemId = queryItemId;
+    this.cteDefinition = cteDefinition;
+    this.programIndicatorUid = null;
+    this.programStageUid = programStageUid;
+    // ignore offset
+    this.alias = new RandomStringGenerator.Builder().withinRange('a', 'z').build().generate(5);
+    this.rowContext = false;
+    this.programIndicator = false;
+    this.filter = isFilter;
+  }
+
+  /**
+   * @param uid the uid of an dimension item or ProgramIndicator
+   * @return the name of the CTE
+   */
+  public String asCteName(String uid) {
+    if (isExists) {
+      return uid.toLowerCase();
+    }
+    if (programIndicator) {
+      return "%s_%s".formatted(PI_PREFIX, programIndicatorUid.toLowerCase());
+    }
+    if (filter) {
+      return uid.toLowerCase();
+    }
+
+    return "%s_%s_%s".formatted(PS_PREFIX, programStageUid.toLowerCase(), uid.toLowerCase());
+  }
+
+  public boolean isProgramStage() {
+    return !filter && !programIndicator && !isExists;
+  }
+
+  public boolean isExists() {
+    return isExists;
+  }
+}