#200: Set CRs to error status if handling was interrupted before

When handling of a resource is unexpectedly interrupted without throwing an error, it currently remains in HANDLING state and is never handled again. This could happen if the operator is shut down mid handling, e.g. because of resource constraints. This commit sets the resource to ERROR state and adds a message that handling was unexpectedly interrupted before. Contributed on behalf of STMicroelectronics Signed-off-by: Lucas Koehler <[email protected]>
eclipse-theia · Aug 30, 2023 · 767824d · 767824d
1 parent 93ed964
commit 767824d
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 13 deletions.
diff --git a/...rator/src/main/java/org/eclipse/theia/cloud/operator/handler/impl/LazySessionHandler.java b/...rator/src/main/java/org/eclipse/theia/cloud/operator/handler/impl/LazySessionHandler.java
@@ -117,12 +117,20 @@ protected boolean doSessionAdded(Session session, String correlationId) {
 		    "Session was successfully handled before and is skipped now. Session: " + session));
 	    return true;
 	}
-	if (OperatorStatus.ERROR.equals(operatorStatus) || OperatorStatus.HANDLING.equals(operatorStatus)) {
-	    // TODO In the HANDLING case we should not return but continue where we left
-	    // off.
+	if (OperatorStatus.HANDLING.equals(operatorStatus)) {
+	    // TODO We should not return but continue where we left off.
 	    LOGGER.warn(formatLogMessage(correlationId,
-		    "Session could not be handled before and is skipped now. Current status: " + operatorStatus
-			    + ". Session: " + session));
+		    "Session handling was unexpectedly interrupted before. Session is skipped now and its status is set to ERROR. Session: "
+			    + session));
+	    client.sessions().updateStatus(correlationId, session, s -> {
+		s.setOperatorStatus(OperatorStatus.ERROR);
+		s.setOperatorMessage("Handling was unexpectedly interrupted before. CorrelationId: " + correlationId);
+	    });
+	    return false;
+	}
+	if (OperatorStatus.ERROR.equals(operatorStatus)) {
+	    LOGGER.warn(formatLogMessage(correlationId,
+		    "Session could not be handled before and is skipped now. Session: " + session));
 	    return false;
 	}
 

diff --git a/...ain/java/org/eclipse/theia/cloud/operator/handler/impl/LazyStartAppDefinitionHandler.java b/...ain/java/org/eclipse/theia/cloud/operator/handler/impl/LazyStartAppDefinitionHandler.java
@@ -71,10 +71,20 @@ protected boolean doAppDefinitionAdded(AppDefinition appDefinition, String corre
 			    + appDefinition));
 	    return true;
 	}
-	if (OperatorStatus.ERROR.equals(operatorStatus) || OperatorStatus.HANDLING.equals(operatorStatus)) {
+	if (OperatorStatus.HANDLING.equals(operatorStatus)) {
+	    // TODO We should not return but continue where we left off.
 	    LOGGER.warn(formatLogMessage(correlationId,
-		    "AppDefinition could not be handled before and is skipped now. Current status: " + operatorStatus
-			    + ". AppDefinition: " + appDefinition));
+		    "AppDefinition handling was unexpectedly interrupted before. AppDefinition is skipped now and its status is set to ERROR. AppDefinition: "
+			    + appDefinition));
+	    client.appDefinitions().updateStatus(correlationId, appDefinition, s -> {
+		s.setOperatorStatus(OperatorStatus.ERROR);
+		s.setOperatorMessage("Handling was unexpectedly interrupted before. CorrelationId: " + correlationId);
+	    });
+	    return false;
+	}
+	if (OperatorStatus.ERROR.equals(operatorStatus)) {
+	    LOGGER.warn(formatLogMessage(correlationId,
+		    "AppDefinition could not be handled before and is skipped now. AppDefinition: " + appDefinition));
 	    return false;
 	}
 

diff --git a/...tor/src/main/java/org/eclipse/theia/cloud/operator/handler/impl/LazyWorkspaceHandler.java b/...tor/src/main/java/org/eclipse/theia/cloud/operator/handler/impl/LazyWorkspaceHandler.java
@@ -69,12 +69,20 @@ protected boolean doWorkspaceAdded(Workspace workspace, String correlationId) {
 		    "Workspace was successfully handled before and is skipped now. Workspace: " + workspace));
 	    return true;
 	}
-	if (OperatorStatus.ERROR.equals(operatorStatus) || OperatorStatus.HANDLING.equals(operatorStatus)) {
-	    // TODO In the HANDLING case we should not return but continue where we left
-	    // off.
+	if (OperatorStatus.HANDLING.equals(operatorStatus)) {
+	    // TODO We should not return but continue where we left off.
 	    LOGGER.warn(formatLogMessage(correlationId,
-		    "Workspace could not be handled before and is skipped now. Current status: " + operatorStatus
-			    + ". Workspace: " + workspace));
+		    "Workspace handling was unexpectedly interrupted before. Workspace is skipped now and its status is set to ERROR. Workspace: "
+			    + workspace));
+	    client.workspaces().updateStatus(correlationId, workspace, s -> {
+		s.setOperatorStatus(OperatorStatus.ERROR);
+		s.setOperatorMessage("Handling was unexpectedly interrupted before. CorrelationId: " + correlationId);
+	    });
+	    return false;
+	}
+	if (OperatorStatus.ERROR.equals(operatorStatus)) {
+	    LOGGER.warn(formatLogMessage(correlationId,
+		    "Workspace could not be handled before and is skipped now. Workspace: " + workspace));
 	    return false;
 	}