feat: add observability (#15)

* add prometheus

* add grafana

* format files

* fix merge conflicts

* fix merge artifacts

* update config formatting

* add observability package

* fix args issue

* fix merge conflicts

* format

* fix errors

* add comments

* update config

* format and prevent aliasing

* update template

* fix typo

* update config

* add extract json from genesis

* correctly template panoptichain config

* fix query issue

* new dashboard

* use dict

* tidy up merge

* add agglayer

* add aggregator

* remove extra panel

* add services dashboard

* update dashboards

* add dashboards

* update readme

* behold!

* add observability args

* update deploy.yml

* update comment

* address comments

.github/workflows/deploy.yml

@@ -52,6 +52,7 @@ jobs:
           yq e '.deploy_cdk_central_environment = false' --inplace params.yml
           yq e '.deploy_cdk_bridge_infra = false' --inplace params.yml
           yq e '.deploy_zkevm_permissionless_node = false' --inplace params.yml
+          yq e '.deploy_observability = false' --inplace params.yml
 
       - name: Deploy L1
         run: |
@@ -88,3 +89,9 @@ jobs:
           yq e '.deploy_zkevm_permissionless_node = true' --inplace params.yml
           kurtosis run --enclave cdk-v1 --args-file params.yml .
           yq e '.deploy_zkevm_permissionless_node = false' --inplace params.yml # reset
+
+      - name: Deploy Observability Stack
+        run: |
+          yq e '.deploy_observability = true' --inplace params.yml
+          kurtosis run --enclave cdk-v1 --args-file params.yml .
+          yq e '.deploy_observability = false' --inplace params.yml # reset

README.org

@@ -135,13 +135,13 @@ Currently, the deployment process includes the following stages:
   3. Deploy ZkEVM Node and CDK Peripheral Databases
   4. Deploy CDK Central/Trusted Environment
   5. Deploy CDK/Bridge Infrastructure
-  6. Deploy Permissioless Node
+  6. Deploy Permissionless Node
 
 Here's an example of how you can specify the stages to run through.
 
 #+begin_src bash
 # Disable all deployment steps.
-yq e 'deploy_l1 = false' --inplace params.yml
+yq e '.deploy_l1 = false' --inplace params.yml
 yq e '.deploy_zkevm_contracts_on_l1 = false' --inplace params.yml
 yq e '.deploy_databases = false' --inplace params.yml
 yq e '.deploy_cdk_central_environment = false' --inplace params.yml

cdk_bridge_infra.star

@@ -247,10 +247,6 @@ def start_dac(plan, args):
             image=args["zkevm_dac_image"],
             ports={
                 "dac": PortSpec(args["zkevm_dac_port"], application_protocol="http"),
-                # Does the DAC have prometheus?!
-                # "prometheus": PortSpec(
-                #     args["zkevm_prometheus_port"], application_protocol="http"
-                # ),
             },
             files={
                 "/etc/zkevm": Directory(

lib/service.star

@@ -5,7 +5,7 @@ def extract_json_key_from_service(plan, service_name, filename, key):
         command=[
             "/bin/sh",
             "-c",
-            "cat {}".format(filename, key),
+            "cat {}".format(filename),
         ],
         extract={"extracted_value": "fromjson | .{}".format(key)},
     )

main.star

@@ -4,6 +4,7 @@ cdk_databases_package = import_module("./cdk_databases.star")
 cdk_central_environment_package = import_module("./cdk_central_environment.star")
 cdk_bridge_infra_package = import_module("./cdk_bridge_infra.star")
 zkevm_permissionless_node_package = import_module("./zkevm_permissionless_node.star")
+observability_package = import_module("./observability.star")
 
 
 def run(plan, args):
@@ -76,11 +77,19 @@ def run(plan, args):
     if args["deploy_zkevm_permissionless_node"]:
         plan.print("Deploying zkevm permissionless node")
         # Note that an additional suffix will be added to the permissionless services.
-        permissionless_node_args = dict(args)  # Create a shallow copy of args.
+        permissionless_node_args = dict(args)
         permissionless_node_args["deployment_suffix"] = (
             "-pless" + args["deployment_suffix"]
         )
         permissionless_node_args["genesis_artifact"] = genesis_artifact
         zkevm_permissionless_node_package.run(plan, permissionless_node_args)
     else:
         plan.print("Skipping the deployment of zkevm permissionless node")
+
+    # Deploy observability stack
+    if args["deploy_observability"]:
+        plan.print("Deploying the observability stack")
+        observability_args = dict(args)
+        observability_package.run(plan, observability_args)
+    else:
+        plan.print("Skipping the deployment of the observability stack")

observability.star

@@ -0,0 +1,110 @@
+prometheus_package = import_module(
+    "github.com/kurtosis-tech/prometheus-package/main.star"
+)
+grafana_package = import_module("github.com/kurtosis-tech/grafana-package/main.star")
+bridge_package = import_module("./cdk_bridge_infra.star")
+service_package = import_module("./lib/service.star")
+
+
+def start_panoptichain(plan, args):
+    # Create the panoptichain config.
+    panoptichain_config_template = read_file(src="./templates/panoptichain-config.yml")
+    panoptichain_config_artifact = plan.render_templates(
+        name="panoptichain-config",
+        config={
+            "config.yml": struct(
+                template=panoptichain_config_template,
+                data={
+                    "l1_rpc_url": args["l1_rpc_url"],
+                    "zkevm_rpc_url": args["zkevm_rpc_url"],
+                    "l1_chain_id": args["l1_chain_id"],
+                    "zkevm_rollup_chain_id": args["zkevm_rollup_chain_id"],
+                    "zkevm_bridge_address": bridge_package.get_key_from_config(
+                        plan, args, "polygonZkEVMBridgeAddress"
+                    ),
+                    "polygon_zkevm_address": bridge_package.get_key_from_config(
+                        plan, args, "rollupAddress"
+                    ),
+                    "rollup_manager_address": bridge_package.get_key_from_config(
+                        plan, args, "polygonRollupManagerAddress"
+                    ),
+                    "global_exit_root_address": bridge_package.get_key_from_config(
+                        plan, args, "polygonZkEVMGlobalExitRootAddress"
+                    ),
+                    "global_exit_root_l2_address": service_package.extract_json_key_from_service(
+                        plan,
+                        "contracts" + args["deployment_suffix"],
+                        "/opt/zkevm/genesis.json",
+                        'genesis[] | select(.contractName == "PolygonZkEVMGlobalExitRootL2 proxy") | .address',
+                    ),
+                    "pol_token_address": bridge_package.get_key_from_config(
+                        plan, args, "polTokenAddress"
+                    ),
+                },
+            )
+        },
+    )
+
+    # Start panoptichain.
+    return plan.add_service(
+        name="panoptichain" + args["deployment_suffix"],
+        config=ServiceConfig(
+            image="minhdvu/panoptichain",
+            ports={
+                "prometheus": PortSpec(9090, application_protocol="http"),
+            },
+            files={"/etc/panoptichain": panoptichain_config_artifact},
+        ),
+    )
+
+
+def run(plan, args):
+    services = []
+    service_names = [
+        "zkevm-agglayer",
+        "zkevm-node-aggregator",
+        "zkevm-node-eth-tx-manager",
+        "zkevm-node-l2-gas-pricer",
+        "zkevm-node-rpc",
+        "zkevm-node-rpc-pless",
+        "zkevm-node-sequence-sender",
+        "zkevm-node-sequencer",
+        "zkevm-node-synchronizer",
+        "zkevm-node-synchronizer-pless",
+    ]
+
+    for name in service_names:
+        service = plan.get_service(name=name + args["deployment_suffix"])
+
+        if not service:
+            continue
+
+        services.append(service)
+        if name == "zkevm-node-rpc":
+            args["zkevm_rpc_url"] = "http://{}:{}".format(
+                service.ip_address, service.ports["http-rpc"].number
+            )
+
+    # Start panoptichain.
+    services.append(start_panoptichain(plan, args))
+
+    metrics_jobs = [
+        {
+            "Name": service.name,
+            "Endpoint": "{0}:{1}".format(
+                service.ip_address,
+                service.ports["prometheus"].number,
+            ),
+        }
+        for service in services
+    ]
+
+    # Start prometheus.
+    prometheus_url = prometheus_package.run(plan, metrics_jobs)
+
+    # Start grafana.
+    grafana_package.run(
+        plan,
+        prometheus_url,
+        "github.com/0xPolygon/kurtosis-cdk/static-files/dashboards",
+    )

params.yml

@@ -25,6 +25,9 @@ deploy_cdk_bridge_infra: true
 # Deploy permissionless node.
 deploy_zkevm_permissionless_node: true
 
+# Deploy observability stack.
+deploy_observability: true
+
 # Docker images and repositories used to spin up services.
 zkevm_prover_image: hermeznetwork/zkevm-prover:v6.0.0
 # zkevm_prover_image: hermeznetwork/zkevm-prover:v4.0.19
@@ -38,7 +41,6 @@ zkevm_fork_id: 9
 # zkevm_fork_id: 7
 zkevm_contracts_repo: https://github.com/0xPolygonHermez/zkevm-contracts.git
 
-
 zkevm_agglayer_image: nulyjkdhthz/agglayer:v0.1.0
 zkevm_bridge_service_image: hermeznetwork/zkevm-bridge-service:v0.4.2
 zkevm_bridge_ui_image: hermeznetwork/zkevm-bridge-ui:multi-network