feat: circuit break on failure

Bisonai · Mar 29, 2024 · ac1f685 · ac1f685
1 parent af69ed7
commit ac1f685
Show file tree

Hide file tree

Showing 5 changed files with 70 additions and 46 deletions.
diff --git a/cli/src/aggregator.ts b/cli/src/aggregator.ts
@@ -282,6 +282,7 @@ export function activateHandler() {
     } catch (e) {
       console.error('Aggregator was not activated. Reason:')
       console.error(e?.response?.data?.message)
+      throw e
     }
   }
   return wrapper
@@ -311,6 +312,7 @@ export function deactivateHandler() {
     } catch (e) {
       console.error('Aggregator was not deactivated. Reason:')
       console.error(e?.response?.data?.message)
+      throw e
     }
   }
   return wrapper

diff --git a/cli/src/datafeed.ts b/cli/src/datafeed.ts
@@ -286,28 +286,36 @@ export function bulkActivateHandler() {
         continue
       }
 
-      await fetcherStartHandler()({
-        id: aggregatorData.aggregatorHash,
-        chain,
-        host: fetcherHost,
-        port: fetcherPort
-      })
-      await aggregatorActivateHandler()({
-        aggregatorHash: aggregatorData.aggregatorHash,
-        host: workerHost,
-        port: workerPort
-      })
-
-      await reporterActivateHandler()({
-        id: Number(reporterId),
-        host: reporterHost,
-        port: reporterPort
-      })
-      await listenerActivateHandler()({
-        id: Number(listenerId),
-        host: listenerHost,
-        port: listenerPort
-      })
+      try {
+        await fetcherStartHandler()({
+          id: aggregatorData.aggregatorHash,
+          chain,
+          host: fetcherHost,
+          port: fetcherPort
+        })
+        await aggregatorActivateHandler()({
+          aggregatorHash: aggregatorData.aggregatorHash,
+          host: workerHost,
+          port: workerPort
+        })
+
+        await reporterActivateHandler()({
+          id: Number(reporterId),
+          host: reporterHost,
+          port: reporterPort
+        })
+        await listenerActivateHandler()({
+          id: Number(listenerId),
+          host: listenerHost,
+          port: listenerPort
+        })
+      } catch (e) {
+        console.error(
+          `activation failed for ${activateElement.aggregatorSource}, breaking iteration`
+        )
+        console.error(e?.response?.data)
+        break
+      }
     }
   }
   return wrapper
@@ -364,30 +372,38 @@ export function bulkDeactivateHandler() {
         continue
       }
 
-      await listenerDeactivateHandler()({
-        id: Number(listenerId),
-        host: listenerHost,
-        port: listenerPort
-      })
-
-      await reporterDeactivateHandler()({
-        id: Number(reporterId),
-        host: reporterHost,
-        port: reporterPort
-      })
-
-      await aggregatorDeactivateHandler()({
-        aggregatorHash: aggregatorData.aggregatorHash,
-        host: workerHost,
-        port: workerPort
-      })
-
-      await fetcherStopHandler()({
-        id: aggregatorData.aggregatorHash,
-        chain,
-        host: fetcherHost,
-        port: fetcherPort
-      })
+      try {
+        await listenerDeactivateHandler()({
+          id: Number(listenerId),
+          host: listenerHost,
+          port: listenerPort
+        })
+
+        await reporterDeactivateHandler()({
+          id: Number(reporterId),
+          host: reporterHost,
+          port: reporterPort
+        })
+
+        await aggregatorDeactivateHandler()({
+          aggregatorHash: aggregatorData.aggregatorHash,
+          host: workerHost,
+          port: workerPort
+        })
+
+        await fetcherStopHandler()({
+          id: aggregatorData.aggregatorHash,
+          chain,
+          host: fetcherHost,
+          port: fetcherPort
+        })
+      } catch (e) {
+        console.error(
+          `deactivation failed for ${deactivateElement.aggregatorSource}, breaking iteration`
+        )
+        console.error(e?.response?.data)
+        break
+      }
     }
   }
   return wrapper

diff --git a/cli/src/fetcher.ts b/cli/src/fetcher.ts
@@ -118,6 +118,7 @@ export function startHandler() {
       console.log(result)
     } catch (e) {
       console.error(e?.response?.data, { depth: null })
+      throw e
     }
   }
   return wrapper
@@ -144,6 +145,7 @@ export function stopHandler() {
       console.log(result)
     } catch (e) {
       console.error(e?.response?.data?.message)
+      throw e
     }
   }
   return wrapper

diff --git a/cli/src/listener.ts b/cli/src/listener.ts
@@ -231,6 +231,7 @@ export function activateHandler() {
     } catch (e) {
       console.error('Listener was not activated. Reason:')
       console.error(e?.response?.data?.message)
+      throw e
     }
   }
   return wrapper
@@ -249,6 +250,7 @@ export function deactivateHandler() {
     } catch (e) {
       console.error('Listener was not deactivated. Reason:')
       console.error(e?.response?.data?.message)
+      throw e
     }
   }
   return wrapper

diff --git a/cli/src/reporter.ts b/cli/src/reporter.ts
@@ -256,6 +256,7 @@ export function activateHandler() {
     } catch (e) {
       console.error('Reporter was not activated. Reason:')
       console.error(e?.response?.data?.message)
+      throw e
     }
   }
   return wrapper
@@ -274,6 +275,7 @@ export function deactivateHandler() {
     } catch (e) {
       console.error('Reporter was not deactivated. Reason:')
       console.error(e?.response?.data?.message)
+      throw e
     }
   }
   return wrapper