fix: 修复熔断规则关闭后，没有清除缓存导致prometheus仍能采集到熔断，半熔断等状态数量 (#442)

polarismesh · Oct 30, 2023 · d4900a0 · d4900a0
1 parent faebe17
commit d4900a0
Show file tree

Hide file tree

Showing 3 changed files with 57 additions and 19 deletions.
diff --git a/...common/polaris-model/src/main/java/com/tencent/polaris/api/pojo/CircuitBreakerStatus.java b/...common/polaris-model/src/main/java/com/tencent/polaris/api/pojo/CircuitBreakerStatus.java
@@ -47,6 +47,11 @@ public class CircuitBreakerStatus {
      */
     private final FallbackInfo fallbackInfo;
 
+    /**
+     * 是否被销毁
+     */
+    private boolean isDestroy;
+
     public CircuitBreakerStatus(String circuitBreaker, Status status, long startTimeMs) {
         this(circuitBreaker, status, startTimeMs, null);
     }
@@ -74,6 +79,14 @@ public FallbackInfo getFallbackInfo() {
         return fallbackInfo;
     }
 
+    public boolean isDestroy() {
+        return isDestroy;
+    }
+
+    public void setDestroy(boolean destroy) {
+        this.isDestroy = destroy;
+    }
+
     /**
      * 是否可以继续分配请求
      *
@@ -120,7 +133,11 @@ public enum Status {
         /**
          * 熔断器打开状态，实例不提供服务
          */
-        OPEN
+        OPEN,
+        /**
+         * 熔断规则被销毁，实例可提供服务
+         */
+        DESTROY
     }
 
 

diff --git a/.../src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/ResourceCounters.java b/.../src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/ResourceCounters.java
@@ -17,8 +17,6 @@
 
 package com.tencent.polaris.plugins.circuitbreaker.composite;
 
-import static com.tencent.polaris.logging.LoggingConsts.LOGGING_CIRCUITBREAKER_EVENT;
-
 import com.tencent.polaris.api.plugin.Plugin;
 import com.tencent.polaris.api.plugin.cache.FlowCache;
 import com.tencent.polaris.api.plugin.circuitbreaker.ResourceStat;
@@ -44,27 +42,20 @@
 import com.tencent.polaris.plugins.circuitbreaker.composite.trigger.ErrRateCounter;
 import com.tencent.polaris.plugins.circuitbreaker.composite.trigger.TriggerCounter;
 import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto;
-import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.CircuitBreakerRule;
-import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.ErrorCondition;
-import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.FallbackConfig;
-import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.FallbackResponse;
+import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.*;
 import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.FallbackResponse.MessageHeader;
-import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.Level;
-import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.TriggerCondition;
 import com.tencent.polaris.specification.api.v1.model.ModelProto.MatchString;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
+import org.slf4j.Logger;
+
+import java.util.*;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 import java.util.regex.Pattern;
-import org.slf4j.Logger;
+
+import static com.tencent.polaris.logging.LoggingConsts.LOGGING_CIRCUITBREAKER_EVENT;
 
 public class ResourceCounters implements StatusChangeHandler {
 
@@ -367,5 +358,21 @@ public void reportCircuitStatus() {
 
     public void setDestroyed(boolean value) {
         destroyed.set(value);
+        toDestroy();
+    }
+
+    private void toDestroy() {
+        synchronized (this) {
+            CircuitBreakerStatus circuitBreakerStatus = circuitBreakerStatusReference.get();
+            circuitBreakerStatus.setDestroy(true);
+            circuitBreakerStatusReference.set(circuitBreakerStatus);
+            CB_EVENT_LOG.info("previous status {}, current status {}, resource {}, rule {}",
+                    circuitBreakerStatus.getStatus(),
+                    Status.DESTROY, resource, circuitBreakerStatus.getCircuitBreaker());
+            for (TriggerCounter triggerCounter : counters) {
+                triggerCounter.resume();
+            }
+            reportCircuitStatus();
+        }
     }
 }
diff --git a/.../tencent/polaris/plugins/stat/common/model/MetricValueAggregationStrategyCollections.java b/.../tencent/polaris/plugins/stat/common/model/MetricValueAggregationStrategyCollections.java
@@ -17,14 +17,14 @@
 
 package com.tencent.polaris.plugins.stat.common.model;
 
-import static com.tencent.polaris.api.pojo.CircuitBreakerStatus.Status.HALF_OPEN;
-import static com.tencent.polaris.api.pojo.CircuitBreakerStatus.Status.OPEN;
-
 import com.tencent.polaris.api.plugin.stat.CircuitBreakGauge;
 import com.tencent.polaris.api.plugin.stat.RateLimitGauge;
 import com.tencent.polaris.api.pojo.InstanceGauge;
 import com.tencent.polaris.api.pojo.RetStatus;
 
+import static com.tencent.polaris.api.pojo.CircuitBreakerStatus.Status.HALF_OPEN;
+import static com.tencent.polaris.api.pojo.CircuitBreakerStatus.Status.OPEN;
+
 public class MetricValueAggregationStrategyCollections {
 
     public static MetricValueAggregationStrategy<InstanceGauge>[] SERVICE_CALL_STRATEGY;
@@ -283,6 +283,15 @@ public void updateMetricValue(StatMetric targetValue, CircuitBreakGauge dataSour
                 return;
             }
 
+            if (targetValue instanceof StatStatefulMetric) {
+                StatStatefulMetric markMetric = ((StatStatefulMetric) targetValue);
+                if (dataSource.getCircuitBreakStatus().isDestroy()) {
+                    targetValue.setValue(0);
+                    markMetric.removeMarkedName(dataSource.getCircuitBreakStatus().getCircuitBreaker());
+                    return;
+                }
+            }
+
             if (OPEN == dataSource.getCircuitBreakStatus().getStatus()) {
                 targetValue.incValue();
             } else if (HALF_OPEN == dataSource.getCircuitBreakStatus().getStatus()) {
@@ -323,6 +332,11 @@ public void updateMetricValue(StatMetric targetValue, CircuitBreakGauge dataSour
 
             if (targetValue instanceof StatStatefulMetric) {
                 StatStatefulMetric markMetric = ((StatStatefulMetric) targetValue);
+                if (dataSource.getCircuitBreakStatus().isDestroy()) {
+                    targetValue.setValue(0);
+                    markMetric.removeMarkedName(dataSource.getCircuitBreakStatus().getCircuitBreaker());
+                    return;
+                }
                 switch (dataSource.getCircuitBreakStatus().getStatus()) {
                     case OPEN:
                         if (markMetric.contain(dataSource.getCircuitBreakStatus().getCircuitBreaker())) {