From 8c8af9859ed7a0587397497e0b1b37e4eab0bd84 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 25 Nov 2024 15:55:56 +0200 Subject: [PATCH 01/88] Initial PR --- clients/erigon/Caddyfile.template | 41 + clients/erigon/Dockerfile | 31 + clients/erigon/config-alloy.template | 206 ++ clients/erigon/erigon.rhai | 39 + clients/reth/Caddyfile.template | 41 + clients/reth/Dockerfile | 36 + clients/reth/config-alloy.template | 111 + clients/reth/reth.rhai | 39 + ethereum/ethereum-erigon/Dockerfile | 3 + ethereum/ethereum-erigon/babel.yaml | 111 + ethereum/ethereum-erigon/main.rhai | 145 + ethereum/ethereum-reth/Dockerfile | 3 + ethereum/ethereum-reth/babel.yaml | 93 + ethereum/ethereum-reth/main.rhai | 112 + node-base/Dockerfile | 83 + node-base/Dockerfile.ubuntu | 51 + node-base/base.rhai | 31 + node-base/config/caddy-dashboard.json | 2361 ++++++++++++++++ node-base/config/grafana-dashboard.yml | 12 + node-base/config/grafana-datasource.yml | 20 + node-base/templates/caddy-dashboard.template | 2366 +++++++++++++++++ node-base/templates/netdata-conf.template | 9 + .../templates/netdata-stream-conf.template | 10 + renovate.json | 52 + 24 files changed, 6006 insertions(+) create mode 100644 clients/erigon/Caddyfile.template create mode 100644 clients/erigon/Dockerfile create mode 100644 clients/erigon/config-alloy.template create mode 100644 clients/erigon/erigon.rhai create mode 100644 clients/reth/Caddyfile.template create mode 100644 clients/reth/Dockerfile create mode 100644 clients/reth/config-alloy.template create mode 100644 clients/reth/reth.rhai create mode 100644 ethereum/ethereum-erigon/Dockerfile create mode 100644 ethereum/ethereum-erigon/babel.yaml create mode 100644 ethereum/ethereum-erigon/main.rhai create mode 100644 ethereum/ethereum-reth/Dockerfile create mode 100644 ethereum/ethereum-reth/babel.yaml create mode 100644 ethereum/ethereum-reth/main.rhai create mode 100644 node-base/Dockerfile create mode 100644 node-base/Dockerfile.ubuntu create mode 100644 node-base/base.rhai create mode 100644 node-base/config/caddy-dashboard.json create mode 100644 node-base/config/grafana-dashboard.yml create mode 100644 node-base/config/grafana-datasource.yml create mode 100644 node-base/templates/caddy-dashboard.template create mode 100644 node-base/templates/netdata-conf.template create mode 100644 node-base/templates/netdata-stream-conf.template create mode 100644 renovate.json diff --git a/clients/erigon/Caddyfile.template b/clients/erigon/Caddyfile.template new file mode 100644 index 00000000..f5bc8a26 --- /dev/null +++ b/clients/erigon/Caddyfile.template @@ -0,0 +1,41 @@ +{ + admin 127.0.0.1:2019 { + } + log { + output stdout + format json + level INFO + } + acme_dns cloudflare UR8KSq7bqAWNJkpol5eEQPiU_K0ADrdXzCZoXEl3 + auto_https disable_redirects + storage file_system {{ data_dir }} + email ops@blockjoy.com + servers { + metrics + } +} + +http://{{ hostname }}{{ tld }}, https://{{ hostname }}{{ tld }}, *.blkjy.io, http:// { + log { + output stdout + } + log_append upstream_host {rp.upstream.host} + log_append upstream_port {rp.upstream.port} + log_append upstream_requests {rp.upstream.requests} + log_append upstream_max_requests {rp.upstream.max_requests} + log_append upstream_fails {rp.upstream.fails} + log_append upstream_latency {rp.upstream.latency} + log_append upstream_latency_ms {rp.upstream.latency_ms} + log_append upstream_duration_ms {rp.upstream.duration_ms} + log_append duration_ms {rp.duration_ms} + @websockets { + header Connection *Upgrade* + header Upgrade websocket + } + reverse_proxy 127.0.0.1:{{ rpc_port }} { + header_up Host localhost + } + reverse_proxy @websockets 127.0.0.1:{{ ws_port }} { + header_up Host localhost + } +} diff --git a/clients/erigon/Dockerfile b/clients/erigon/Dockerfile new file mode 100644 index 00000000..756dc23d --- /dev/null +++ b/clients/erigon/Dockerfile @@ -0,0 +1,31 @@ +FROM ghcr.io/blockjoy/node-base:abc123 + +# Add build arguments for Grafana credentials +ARG GRAFANA_LOKI_BASICAUTH +ARG GRAFANA_PROM_BASICAUTH + +ENV DEBIAN_FRONTEND=noninteractive +ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} +ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} + +ENV ERIGON_VERSION v2.60.9 + +ENV PATH="/usr/local/go/bin:${PATH}" +ENV GOROOT="/usr/local/go" + +RUN mkdir -p /root/bin + +WORKDIR /root +RUN git clone https://github.com/erigontech/erigon.git && \ + cd erigon && \ + git checkout $ERIGON_VERSION + +WORKDIR /root/erigon +RUN make erigon + +RUN cp /root/erigon/build/bin/erigon /root/bin/erigon && \ + rm -rf /root/erigon + +COPY Caddyfile.template /var/lib/babel/templates/Caddyfile.template +COPY config-alloy.template /var/lib/babel/templates/config-alloy.template +COPY erigon.rhai /var/lib/babel/plugin/erigon.rhai diff --git a/clients/erigon/config-alloy.template b/clients/erigon/config-alloy.template new file mode 100644 index 00000000..0734ac5d --- /dev/null +++ b/clients/erigon/config-alloy.template @@ -0,0 +1,206 @@ +logging { + level = "warn" +} + +discovery.relabel "metrics_grafana_agent" { + targets = [{ + __address__ = "127.0.0.1:12346", + }] + + rule { + target_label = "instance" + replacement = "{{ hostname }}{{ tld }}" + } +} + +prometheus.scrape "metrics_grafana_agent" { + targets = discovery.relabel.metrics_grafana_agent.output + forward_to = [prometheus.remote_write.default.receiver] + job_name = "grafana_agent" + metrics_path = "/metrics" +} + +discovery.relabel "metrics_caddy" { + targets = [{ + __address__ = "127.0.0.1:2019", + }] + + rule { + target_label = "instance" + replacement = "{{ hostname }}{{ tld }}" + } +} + +prometheus.scrape "metrics_caddy" { + targets = discovery.relabel.metrics_caddy.output + forward_to = [prometheus.remote_write.default.receiver] + job_name = "caddy" + metrics_path = "/metrics" +} + +discovery.relabel "metrics_{{ blockchain | lower | replace(from="-", to="_") }}" { + targets = [{ + __address__ = "127.0.0.1:{{ metrics_port }}", + }] + + rule { + target_label = "instance" + replacement = "{{ hostname }}{{ tld }}" + } + + rule { + target_label = "protocol" + replacement = "{{ blockchain | lower | replace(from="-", to="_") }}" + } +} + +prometheus.scrape "metrics_{{ blockchain | lower | replace(from="-", to="_") }}" { + targets = discovery.relabel.metrics_{{ blockchain | lower | replace(from="-", to="_") }}.output + forward_to = [prometheus.remote_write.default.receiver] + job_name = "{{ blockchain | lower | replace(from="-", to="_") }}" + metrics_path = "{{ metrics_path }}" +} + +local.file_match "babel" { + path_targets = [{ + __address__ = "127.0.0.1", + __path__ = "/var/lib/babel/jobs/logs/*", + instance = "{{ hostname }}{{ tld }}", + job = "babel", + }] +} + +loki.source.file "logs_default_babel" { + targets = local.file_match.babel.targets + forward_to = [loki.write.logs_default.receiver] +} + + +discovery.relabel "logs_default_journal_scrape" { + targets = [] + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + + rule { + source_labels = ["__journal__boot_id"] + target_label = "boot_id" + } + + rule { + source_labels = ["__journal__transport"] + target_label = "transport" + } + + rule { + source_labels = ["__journal_priority_keyword"] + target_label = "level" + } +} + +loki.source.journal "logs_default_journal_scrape" { + max_age = "24h0m0s" + relabel_rules = discovery.relabel.logs_default_journal_scrape.rules + forward_to = [loki.write.logs_default.receiver,loki.write.logs_local.receiver] + labels = { + cluster = "prod-nodes", + instance = "{{ hostname }}{{ tld }}", + job = "systemd_logs", + } +} + +loki.write "logs_default" { + endpoint { + url = "https://logs-prod-012.grafana.net/loki/api/v1/push" + + basic_auth { + username = "760471" + password = ${GRAFANA_LOKI_BASICAUTH} + } + } + external_labels = {} +} + +loki.write "logs_local" { + endpoint { + url = "http://localhost:9090/loki/api/v1/push" + } + external_labels = {} +} + +prometheus.exporter.unix "node_exporter" { + disable_collectors = ["ipvs", "btrfs", "infiniband", "xfs", "zfs"] + + filesystem { + fs_types_exclude = "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|tmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$" + mount_points_exclude = "^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)" + mount_timeout = "5s" + } + + netclass { + ignored_devices = "^(veth.*|cali.*|[a-f0-9]{15})$" + } + + netdev { + device_exclude = "^(veth.*|cali.*|[a-f0-9]{15})$" + } +} + +discovery.relabel "node_exporter" { + targets = prometheus.exporter.unix.node_exporter.targets + + + rule { + target_label = "instance" + replacement = "{{ hostname }}{{ tld }}" + } + + rule { + target_label = "job" + replacement = "node_exporter" + } +} + +prometheus.scrape "node_exporter" { + targets = discovery.relabel.node_exporter.output + forward_to = [prometheus.relabel.node_exporter.receiver] + job_name = "node_exporter" +} + +prometheus.relabel "node_exporter" { + forward_to = [prometheus.remote_write.default.receiver] + + rule { + source_labels = ["__name__"] + regex = "node_scrape_collector_.+" + action = "drop" + } + + rule { + source_labels = ["__name__"] + regex = "node_arp_entries|node_boot_time_seconds|node_context_switches_total|node_cpu_seconds_total|node_disk_io_time_seconds_total|node_disk_io_time_weighted_seconds_total|node_disk_read_bytes_total|node_disk_read_time_seconds_total|node_disk_reads_completed_total|node_disk_write_time_seconds_total|node_disk_writes_completed_total|node_disk_written_bytes_total|node_filefd_allocated|node_filefd_maximum|node_filesystem_avail_bytes|node_filesystem_device_error|node_filesystem_files|node_filesystem_files_free|node_filesystem_readonly|node_filesystem_size_bytes|node_intr_total|node_load1|node_load15|node_load5|node_md_disks|node_md_disks_required|node_memory_Active_anon_bytes|node_memory_Active_bytes|node_memory_Active_file_bytes|node_memory_AnonHugePages_bytes|node_memory_AnonPages_bytes|node_memory_Bounce_bytes|node_memory_Buffers_bytes|node_memory_Cached_bytes|node_memory_CommitLimit_bytes|node_memory_Committed_AS_bytes|node_memory_DirectMap1G_bytes|node_memory_DirectMap2M_bytes|node_memory_DirectMap4k_bytes|node_memory_Dirty_bytes|node_memory_HugePages_Free|node_memory_HugePages_Rsvd|node_memory_HugePages_Surp|node_memory_HugePages_Total|node_memory_Hugepagesize_bytes|node_memory_Inactive_anon_bytes|node_memory_Inactive_bytes|node_memory_Inactive_file_bytes|node_memory_Mapped_bytes|node_memory_MemAvailable_bytes|node_memory_MemFree_bytes|node_memory_MemTotal_bytes|node_memory_SReclaimable_bytes|node_memory_SUnreclaim_bytes|node_memory_ShmemHugePages_bytes|node_memory_ShmemPmdMapped_bytes|node_memory_Shmem_bytes|node_memory_Slab_bytes|node_memory_SwapTotal_bytes|node_memory_VmallocChunk_bytes|node_memory_VmallocTotal_bytes|node_memory_VmallocUsed_bytes|node_memory_WritebackTmp_bytes|node_memory_Writeback_bytes|node_netstat_Icmp6_InErrors|node_netstat_Icmp6_InMsgs|node_netstat_Icmp6_OutMsgs|node_netstat_Icmp_InErrors|node_netstat_Icmp_InMsgs|node_netstat_Icmp_OutMsgs|node_netstat_IpExt_InOctets|node_netstat_IpExt_OutOctets|node_netstat_TcpExt_ListenDrops|node_netstat_TcpExt_ListenOverflows|node_netstat_TcpExt_TCPSynRetrans|node_netstat_Tcp_InErrs|node_netstat_Tcp_InSegs|node_netstat_Tcp_OutRsts|node_netstat_Tcp_OutSegs|node_netstat_Tcp_RetransSegs|node_netstat_Udp6_InDatagrams|node_netstat_Udp6_InErrors|node_netstat_Udp6_NoPorts|node_netstat_Udp6_OutDatagrams|node_netstat_Udp6_RcvbufErrors|node_netstat_Udp6_SndbufErrors|node_netstat_UdpLite_InErrors|node_netstat_Udp_InDatagrams|node_netstat_Udp_InErrors|node_netstat_Udp_NoPorts|node_netstat_Udp_OutDatagrams|node_netstat_Udp_RcvbufErrors|node_netstat_Udp_SndbufErrors|node_network_carrier|node_network_info|node_network_mtu_bytes|node_network_receive_bytes_total|node_network_receive_compressed_total|node_network_receive_drop_total|node_network_receive_errs_total|node_network_receive_fifo_total|node_network_receive_multicast_total|node_network_receive_packets_total|node_network_speed_bytes|node_network_transmit_bytes_total|node_network_transmit_compressed_total|node_network_transmit_drop_total|node_network_transmit_errs_total|node_network_transmit_fifo_total|node_network_transmit_multicast_total|node_network_transmit_packets_total|node_network_transmit_queue_length|node_network_up|node_nf_conntrack_entries|node_nf_conntrack_entries_limit|node_os_info|node_sockstat_FRAG6_inuse|node_sockstat_FRAG_inuse|node_sockstat_RAW6_inuse|node_sockstat_RAW_inuse|node_sockstat_TCP6_inuse|node_sockstat_TCP_alloc|node_sockstat_TCP_inuse|node_sockstat_TCP_mem|node_sockstat_TCP_mem_bytes|node_sockstat_TCP_orphan|node_sockstat_TCP_tw|node_sockstat_UDP6_inuse|node_sockstat_UDPLITE6_inuse|node_sockstat_UDPLITE_inuse|node_sockstat_UDP_inuse|node_sockstat_UDP_mem|node_sockstat_UDP_mem_bytes|node_sockstat_sockets_used|node_softnet_dropped_total|node_softnet_processed_total|node_softnet_times_squeezed_total|node_systemd_unit_state|node_textfile_scrape_error|node_time_zone_offset_seconds|node_timex_estimated_error_seconds|node_timex_maxerror_seconds|node_timex_offset_seconds|node_timex_sync_status|node_uname_info|node_vmstat_oom_kill|node_vmstat_pgfault|node_vmstat_pgmajfault|node_vmstat_pgpgin|node_vmstat_pgpgout|node_vmstat_pswpin|node_vmstat_pswpout|process_max_fds|process_open_fds" + action = "keep" + } +} + +prometheus.remote_write "default" { + external_labels = { + cluster = "prod-nodes", + } + + endpoint { + name = "prom-ep" + url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push" + + basic_auth { + username = "1322871" + password = ${GRAFANA_PROM_BASICAUTH} + } + + queue_config { } + + metadata_config { } + } +} diff --git a/clients/erigon/erigon.rhai b/clients/erigon/erigon.rhai new file mode 100644 index 00000000..153dae00 --- /dev/null +++ b/clients/erigon/erigon.rhai @@ -0,0 +1,39 @@ +fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { + #{ + config_files: [ + #{ + template: "/var/lib/babel/templates/config-alloy.template", + destination: "/etc/alloy/config.alloy", + params: #{ + hostname: node_env().node_name, + tld: ".n0des.xyz", + blockchain: node_env().node_protocol, + metrics_port: `${metrics_port}`, + metrics_path: "/debug/metrics/prometheus", + } + }, + #{ + template: "/var/lib/babel/templates/Caddyfile.template", + destination: "/etc/caddy/Caddyfile", + params: #{ + rpc_port: `${rpc_port}`, + ws_port: `${ws_port}`, + hostname: node_env().node_name, + node_ip: node_env().node_ip, + tld: ".n0des.xyz", + data_dir: `${caddy_dir}`, + } + } + ], + services: [ + #{ + name: "alloy", + run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" + }, + #{ + name: "caddy", + run_sh: "/usr/bin/caddy run --config /etc/caddy/Caddyfile" + } + ] + } +} diff --git a/clients/reth/Caddyfile.template b/clients/reth/Caddyfile.template new file mode 100644 index 00000000..91fb841f --- /dev/null +++ b/clients/reth/Caddyfile.template @@ -0,0 +1,41 @@ +{ + admin 127.0.0.1:2019 { + } + log { + output stdout + format json + level INFO + } + acme_dns cloudflare UR8KSq7bqAWNJkpol5eEQPiU_K0ADrdXzCZoXEl3 + auto_https disable_redirects + storage file_system {{ data_dir }} + email ops@blockjoy.com + servers { + metrics + } +} + +http://{{ hostname }}{{ tld }}, https://{{ hostname }}{{ tld }}, *.blkjy.io, http:// { + log { + output stdout + } + log_append upstream_host {rp.upstream.host} + log_append upstream_port {rp.upstream.port} + log_append upstream_requests {rp.upstream.requests} + log_append upstream_max_requests {rp.upstream.max_requests} + log_append upstream_fails {rp.upstream.fails} + log_append upstream_latency {rp.upstream.latency} + log_append upstream_latency_ms {rp.upstream.latency_ms} + log_append upstream_duration_ms {rp.upstream.duration_ms} + log_append duration_ms {rp.duration_ms} + @websockets { + header Connection *Upgrade* + header Upgrade websocket + } + reverse_proxy 127.0.0.1:{{ rpc_port }} { + header_up Host localhost + } + reverse_proxy @websockets 127.0.0.1:{{ ws_port }} { + header_up Host localhost + } +} \ No newline at end of file diff --git a/clients/reth/Dockerfile b/clients/reth/Dockerfile new file mode 100644 index 00000000..c64e998c --- /dev/null +++ b/clients/reth/Dockerfile @@ -0,0 +1,36 @@ +FROM node-base:0.0.1 + +ENV DEBIAN_FRONTEND=noninteractive + +# Install build dependencies +RUN apt-get update && apt-get install -y \ + clang \ + libclang-dev \ + llvm-dev \ + && rm -rf /var/lib/apt/lists/* + +ENV RETH_VERSION=v1.1.2 +ENV CARGO_NET_GIT_FETCH_WITH_CLI=true + +# Install Rust and required dependencies +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \ + . "$HOME/.cargo/env" && \ + rustup default stable && \ + rustup target add x86_64-unknown-linux-gnu + +WORKDIR /root +RUN git clone https://github.com/paradigmxyz/reth.git && \ + cd reth && \ + git checkout $RETH_VERSION + +# Build reth with performance features +WORKDIR /root/reth +RUN . "$HOME/.cargo/env" && \ + mkdir /root/bin && \ + cargo build --profile maxperf && \ + cp target/maxperf/reth /root/bin/ && \ + rm -rf /root/reth + +COPY Caddyfile.template /var/lib/babel/templates/Caddyfile.template +COPY config-alloy.template /var/lib/babel/templates/config-alloy.template +COPY reth.rhai /var/lib/babel/plugin/reth.rhai diff --git a/clients/reth/config-alloy.template b/clients/reth/config-alloy.template new file mode 100644 index 00000000..56d348bf --- /dev/null +++ b/clients/reth/config-alloy.template @@ -0,0 +1,111 @@ +logging { + level = "warn" +} + +discovery.relabel "metrics_grafana_agent" { + targets = [{ + __address__ = "127.0.0.1:12346", + }] + + rule { + target_label = "instance" + replacement = "{{ hostname }}{{ tld }}" + } +} + +prometheus.scrape "metrics_grafana_agent" { + targets = discovery.relabel.metrics_grafana_agent.output + forward_to = [prometheus.remote_write.default.receiver] + job_name = "grafana_agent" + metrics_path = "/metrics" +} + +discovery.relabel "metrics_caddy" { + targets = [{ + __address__ = "127.0.0.1:2019", + }] + + rule { + target_label = "instance" + replacement = "{{ hostname }}{{ tld }}" + } +} + +prometheus.scrape "metrics_caddy" { + targets = discovery.relabel.metrics_caddy.output + forward_to = [prometheus.remote_write.default.receiver] + job_name = "caddy" + metrics_path = "/metrics" +} + +discovery.relabel "metrics_{{ blockchain | lower | replace(from="-", to="_") }}" { + targets = [{ + __address__ = "127.0.0.1:{{ metrics_port }}", + }] + + rule { + target_label = "instance" + replacement = "{{ hostname }}{{ tld }}" + } + + rule { + target_label = "protocol" + replacement = "{{ blockchain | lower | replace(from="-", to="_") }}" + } +} + +prometheus.scrape "metrics_{{ blockchain | lower | replace(from="-", to="_") }}" { + targets = discovery.relabel.metrics_{{ blockchain | lower | replace(from="-", to="_") }}.output + forward_to = [prometheus.remote_write.default.receiver] + job_name = "{{ blockchain | lower | replace(from="-", to="_") }}" + metrics_path = "/{{ blockchain | lower | replace(from="-", to="_") }}/metrics" +} + +local.file_match "babel" { + path_targets = [{ + __address__ = "127.0.0.1", + __path__ = "/var/lib/babel/jobs/logs/*", + instance = "{{ hostname }}{{ tld }}", + job = "babel", + }] +} + +loki.source.file "logs_default_babel" { + targets = local.file_match.babel.targets + forward_to = [loki.write.logs_default.receiver] +} + +discovery.relabel "logs_default_journal_scrape" { + targets = [] + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + + rule { + source_labels = ["__journal__boot_id"] + target_label = "boot_id" + } + + rule { + source_labels = ["__journal__transport"] + target_label = "transport" + } + + rule { + source_labels = ["__journal_priority_keyword"] + target_label = "level" + } +} + +prometheus.remote_write "default" { + endpoint { + url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push" + + basic_auth { + username = "1234567" + password = ${GRAFANA_PROM_BASICAUTH} + } + } +} diff --git a/clients/reth/reth.rhai b/clients/reth/reth.rhai new file mode 100644 index 00000000..b199716c --- /dev/null +++ b/clients/reth/reth.rhai @@ -0,0 +1,39 @@ +fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { + #{ + config_files: [ + #{ + template: "/var/lib/babel/templates/config-alloy.template", + destination: "/etc/alloy/config.alloy", + params: #{ + hostname: node_env().node_name, + tld: ".n0des.xyz", + blockchain: node_env().node_protocol, + metrics_port: `${metrics_port}`, + metrics_path: "/metrics", + } + }, + #{ + template: "/var/lib/babel/templates/Caddyfile.template", + destination: "/etc/caddy/Caddyfile", + params: #{ + rpc_port: `${rpc_port}`, + ws_port: `${ws_port}`, + hostname: node_env().node_name, + node_ip: node_env().node_ip, + tld: ".n0des.xyz", + data_dir: `${caddy_dir}`, + } + } + ], + services: [ + #{ + name: "alloy", + run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" + }, + #{ + name: "caddy", + run_sh: "/usr/bin/caddy run --config /etc/caddy/Caddyfile" + } + ] + } +} \ No newline at end of file diff --git a/ethereum/ethereum-erigon/Dockerfile b/ethereum/ethereum-erigon/Dockerfile new file mode 100644 index 00000000..d3e858ad --- /dev/null +++ b/ethereum/ethereum-erigon/Dockerfile @@ -0,0 +1,3 @@ +FROM ghcr.io/blockjoy/blockjoy-erigon:def456 + +COPY ./main.rhai /var/lib/babel/plugin/ diff --git a/ethereum/ethereum-erigon/babel.yaml b/ethereum/ethereum-erigon/babel.yaml new file mode 100644 index 00000000..9f1f89a6 --- /dev/null +++ b/ethereum/ethereum-erigon/babel.yaml @@ -0,0 +1,111 @@ +version: 0.0.1 +container_uri: docker-daemon://ethereum-erigon:latest +sku_code: ETH-ERG +org_id: null +description: Ethereum Erigon node +visibility: public +protocol_key: ethereum +variants: + # The same protocol can have different implementations. This field let identify what implementation(s) has been used. + # Variant name may additional contain protocol network name e.g. "eth-mainnet". + # This value will be available in a Rhai runtime via `node_env().variant_key`. + - key: erigon-mainnet-archive + archive_pointers: + - pointer: !store_id ethereum-erigon-mainnet-archive-v1 + - pointer: !store_id ethereum-erigon-mainnet-archive-v1 + new_archive_properties: [ "tracing-api-on-value" ] + min_cpu: 8 + min_memory_mb: 32000 + min_disk_gb: 5000 + - key: erigon-sepolia-archive + archive_pointers: + - pointer: !store_id ethereum-erigon-sepolia-archive-v1 + - pointer: !store_id ethereum-erigon-sepolia-archive-v1 + new_archive_properties: [ "tracing-api-on-value" ] + min_cpu: 4 + min_memory_mb: 16000 + min_disk_gb: 2000 + - key: erigon-goerli-archive + archive_pointers: + - pointer: !store_id ethereum-erigon-goerli-archive-v1 + - pointer: !store_id ethereum-erigon-goerli-archive-v1 + new_archive_properties: [ "tracing-api-on-value" ] + min_cpu: 4 + min_memory_mb: 16000 + min_disk_gb: 2000 + - key: erigon-mainnet-full + archive_pointers: + - pointer: !store_id ethereum-erigon-mainnet-full-v1 + - pointer: !combination_disallowed + new_archive_properties: [ "tracing-api-on-value" ] + min_cpu: 8 + min_memory_mb: 32000 + min_disk_gb: 5000 + - key: erigon-sepolia-full + archive_pointers: + - pointer: !store_id ethereum-erigon-sepolia-full-v1 + - pointer: !combination_disallowed + new_archive_properties: [ "tracing-api-on-value" ] + min_cpu: 4 + min_memory_mb: 16000 + min_disk_gb: 2000 + - key: erigon-goerli-full + archive_pointers: + - pointer: !store_id ethereum-erigon-goerli-full-v1 + - pointer: !combination_disallowed + new_archive_properties: [ "tracing-api-on-value" ] + min_cpu: 4 + min_memory_mb: 16000 + min_disk_gb: 2000 +firewall_config: + default_in: deny + default_out: allow + rules: + - key: Allow incoming ports TCP + description: Allow TCP inbound traffic on port 80, 443 + protocol: tcp + direction: in + action: allow + ips: + - ip: 0.0.0.0/0 + name: allow all traffic + ports: + - port: 80 + name: HTTP port + - port: 443 + name: HTTPS port + - port: 3000 + name: Grafana port + - key: Allow P2P ports TCP/UDP + description: Allow inbound traffic on P2P ports + protocol: both + direction: in + action: allow + ips: + - ip: 0.0.0.0/0 + name: allow all traffic + ports: + - port: 30303 + name: P2P port 1 + - port: 30304 + name: P2P port 2 + - port: 42069 + name: Torrent port +properties: + - key: tracing-api + description: This enables or disables the tracing API + dynamic_value: false + default_value: off-value + ui_type: !switch + on: + value: on-value + impact: + new_archive: true + add_cpu: 3 + add_memory_bytes: 10000000 + add_disk_bytes: 10000000000 + off: + value: off-value + impact: null + # Unique property key. Property value will be available in a Rhai runtime via `node_params().` (e.g. `node_params().arbitrary_property`). + # It is required to be lower-kebab-case. \ No newline at end of file diff --git a/ethereum/ethereum-erigon/main.rhai b/ethereum/ethereum-erigon/main.rhai new file mode 100644 index 00000000..6119d70e --- /dev/null +++ b/ethereum/ethereum-erigon/main.rhai @@ -0,0 +1,145 @@ +import "base" as base; +import "erigon" as erigon; + +const CADDY_DIR = DATA_DRIVE_MOUNT_POINT + "/caddy"; +const RPC_PORT = 8545; +const WS_PORT = 8546; +const METRICS_PORT = 6161; + +let ERIGON_CONFIG = erigon::base_config(global::METRICS_PORT,global::RPC_PORT,global::WS_PORT,global::CADDY_DIR); + +const BASE_CONFIG = #{ + config_files: ERIGON_CONFIG.config_files+base::BASE_CONFIG.config_files, + services: ERIGON_CONFIG.services+base::BASE_CONFIG.services, +}; + +const VARIANTS = #{ + erigon_goerli_full: #{ + net: "goerli", + }, + erigon_sepolia_full: #{ + net: "sepolia", + }, + erigon_mainnet_full: #{ + net: "mainnet", + }, + erigon_goerli_archive: #{ + net: "goerli", + }, + erigon_sepolia_archive: #{ + net: "sepolia", + }, + erigon_mainnet_archive: #{ + net: "mainnet", + }, +}; + + +const API_HOST = `http://127.0.0.1:${global::RPC_PORT}`; + +const HOME = PROTOCOL_DATA_PATH + "/erigon"; + +const NET = VARIANTS[node_env().node_variant].net; + +const EXTRA_ERIGON_ARGS = if node_params().nodetype == "full" { + "--prune=htcr" +} else { + "" +}; + +const BABEL_VERSION = "0.9.0"; + +const PLUGIN_CONFIG = #{ + init: #{ + commands: [ + `mkdir -p ${global::HOME}`, + `mkdir -p ${global::CADDY_DIR}`, + ], + jobs: [ + #{ + name: "create_jwt", + run_sh: `export HOME=/root && openssl rand -hex 32 | tr -d "\n" > ${global::HOME}/jwt.hex`, + } + ] + }, + services: [ + #{ + name: "erigon", + run_sh: `export HOME=/root && /root/bin/erigon --datadir=${global::HOME} \ + --chain=${global::NET} --healthcheck=true --port=30303 --http.port=8545 --authrpc.port=8551 \ + --torrent.port=42069 --private.api.addr=127.0.0.1:9090 --http --ws --ws.port 8546 \ + --http.api=eth,debug,net,trace,web3,erigon --nat=none --authrpc.jwtsecret=${global::HOME}/jwt.hex \ + --torrent.download.rate=100mb --metrics --metrics.port=6666 --metrics.addr=127.0.0.1 \ + --http.vhosts localhost --downloader.disable.ipv6 --caplin.backfilling=true \ + --caplin.backfilling.blob=true --caplin.backfilling.blob.no-pruning=true --caplin.archive=true \ + --beacon.api=beacon,builder,config,debug,node,validator,lighthouse --internalcl \ + ${global::EXTRA_ERIGON_ARGS}`, + shutdown_timeout_secs: 120, + use_blockchain_data: true, + }, + ], + alternative_download: #{ + run_sh: "echo dummy" + }, + upload: #{ + exclude: [ + "caddy/**", + ] + } +}; + +fn application_status() { + let resp = run_jrpc(#{host: global::API_HOST, method: ""}); + + if resp.status_code == 200 { + "broadcasting" + } else { + "delinquent" + } +} + +fn address() { + parse_json(run_jrpc(global::API_HOST, "eth_coinbase")).result.to_string() +} + +fn height() { + let resp = run_jrpc(#{host: global::API_HOST, method: "eth_blockNumber"}); + if resp.status_code != 200 { + throw resp; + } + let h = parse_json(resp.body); + parse_int(sub_string(h.result,2),16) +} + +fn sync_status() { + let data1 = #{url: global::API_HOST+"/health", + method: "eth_getBlockByNumber", + headers: [ + ["X-ERIGON-HEALTHCHECK", "synced"], + ["X-ERIGON-HEALTHCHECK", "min_peer_count10"] + ] + }; + + let resp1 = run_rest(data1); + + let data2 = #{host: global::API_HOST, + method: "eth_getBlockByNumber", + params: ["latest", true], + headers: [["content-type", "application/json"]], + }; + + let t = run_jrpc(data2).expect(200); + + let block_timestamp = parse_hex(t.result.timestamp); + + let system_timestamp = run_sh (`date +%s`).unwrap(); + let current_timestamp = parse_int(system_timestamp); + + if resp1.status_code > 225 { + "syncing" + } else if (block_timestamp < current_timestamp - 100) { + "syncing" + } else { + "synced" + } +} diff --git a/ethereum/ethereum-reth/Dockerfile b/ethereum/ethereum-reth/Dockerfile new file mode 100644 index 00000000..98024573 --- /dev/null +++ b/ethereum/ethereum-reth/Dockerfile @@ -0,0 +1,3 @@ +FROM reth:v1.1.2 + +COPY ./main.rhai /var/lib/babel/plugin/ diff --git a/ethereum/ethereum-reth/babel.yaml b/ethereum/ethereum-reth/babel.yaml new file mode 100644 index 00000000..e06aad11 --- /dev/null +++ b/ethereum/ethereum-reth/babel.yaml @@ -0,0 +1,93 @@ +version: 0.0.1 +container_uri: docker-daemon://ethereum-reth:latest +sku_code: ETH-RETH +org_id: null +description: Ethereum Reth node +visibility: public +protocol_key: ethereum +variants: + # The same protocol can have different implementations. This field let identify what implementation(s) has been used. + # Variant name may additional contain protocol network name e.g. "eth-mainnet". + # This value will be available in a Rhai runtime via `node_env().variant_key`. + - key: reth_mainnet_archive + min_cpu: 12 + min_memory_mb: 32000 + min_disk_gb: 3500 + archive_pointers: + - pointer: !store_id ethereum-reth-mainnet-archive-v1 + - pointer: !store_id ethereum-reth-mainnet-archive-v1 + new_archive_properties: [ "tracing-api-on-value" ] + - key: reth_sepolia_archive + min_cpu: 6 + min_memory_mb: 16000 + min_disk_gb: 1000 + archive_pointers: + - pointer: !store_id ethereum-reth-sepolia-archive-v1 + - pointer: !store_id ethereum-reth-sepolia-archive-v1 + new_archive_properties: [ "tracing-api-on-value" ] + - key: reth_mainnet_full + min_cpu: 8 + min_memory_mb: 16000 + min_disk_gb: 1500 + archive_pointers: + - pointer: !store_id ethereum-reth-mainnet-full-v1 + - pointer: !combination_disallowed + new_archive_properties: [ "tracing-api-on-value" ] + - key: reth_sepolia_full + min_cpu: 4 + min_memory_mb: 8000 + min_disk_gb: 400 + archive_pointers: + - pointer: !store_id ethereum-reth-sepolia-full-v1 + - pointer: !combination_disallowed + new_archive_properties: [ "tracing-api-on-value" ] +firewall_config: + default_in: deny + default_out: allow + rules: + - key: Allow incoming ports TCP + description: Allow TCP inbound traffic on port 80, 443 + protocol: tcp + direction: in + action: allow + ips: + - ip: 0.0.0.0/0 + name: allow all traffic + ports: + - port: 80 + name: HTTP port + - port: 443 + name: HTTPS port + - port: 3000 + name: Grafana port + - key: Allow P2P ports TCP/UDP + description: Allow inbound traffic on P2P ports + protocol: both + direction: in + action: allow + ips: + - ip: 0.0.0.0/0 + name: allow all traffic + ports: + - port: 30303 + name: P2P port + - port: 42069 + name: Torrent port +properties: + # Property values will be available in a Rhai runtime via `node_params().` (e.g. `node_params().tracing-api`). + # It is required to be lower-kebab-case. + - key: tracing-api + description: This enables or disables the tracing API + dynamic_value: false + default_value: off-value + ui_type: !switch + on: + value: on-value + impact: + new_archive: true + add_cpu: 3 + add_memory_bytes: 10000000 + add_disk_bytes: 10000000000 + off: + value: off-value + impact: null \ No newline at end of file diff --git a/ethereum/ethereum-reth/main.rhai b/ethereum/ethereum-reth/main.rhai new file mode 100644 index 00000000..813f5faf --- /dev/null +++ b/ethereum/ethereum-reth/main.rhai @@ -0,0 +1,112 @@ +import "base" as base; +import "reth" as reth; + +const CADDY_DIR = DATA_DRIVE_MOUNT_POINT + "/caddy"; +const RPC_PORT = 8545; +const WS_PORT = 8546; +const METRICS_PORT = 6665; + +let RETH_CONFIG = reth::base_config(global::METRICS_PORT,global::RPC_PORT,global::WS_PORT,global::CADDY_DIR); + +const BASE_CONFIG = #{ + config_files: RETH_CONFIG.config_files+base::BASE_CONFIG.config_files, + services: RETH_CONFIG.services+base::BASE_CONFIG.services, +}; + +const VARIANTS = #{ + reth_sepolia_full: #{ + net: "sepolia", + }, + reth_mainnet_full: #{ + net: "mainnet", + }, + reth_sepolia_archive: #{ + net: "sepolia", + }, + reth_mainnet_archive: #{ + net: "mainnet", + }, +}; + +const API_HOST = `http://127.0.0.1:${global::RPC_PORT}`; +const HOME = PROTOCOL_DATA_PATH + "/reth"; +const NET = VARIANTS[node_env().node_variant].net; +const CADDY_DIR = DATA_DRIVE_MOUNT_POINT + "/caddy"; + +const RETH_EXTRA_ARGS = if node_params().nodetype == "full" { + "--db-mode=full --pruning.block-limit=1048576" +} else { + "--db-mode=archive" +}; + +const BABEL_VERSION = "0.9.0"; + +const PLUGIN_CONFIG = #{ + init: #{ + commands: [ + `mkdir -p ${global::HOME}`, + `mkdir -p ${global::CADDY_DIR}`, + ], + jobs: [ + #{ + name: "create_jwt", + run_sh: `export HOME=/root && openssl rand -hex 32 | tr -d "\n" > ${global::HOME}/jwt.hex`, + } + ] + }, + services: [ + #{ + name: "reth", + run_sh: `export HOME=/root && /root/bin/reth node \ + --datadir=${global::HOME} \ + --chain=${global::NET} \ + --http \ + --http.addr=127.0.0.1 \ + --http.port=${global::RPC_PORT} \ + --http.api=eth,net,web3,debug,trace,txpool \ + --ws \ + --ws.addr=127.0.0.1 \ + --ws.port=${global::WS_PORT} \ + --ws.api=eth,net,web3,debug,trace,txpool \ + --metrics=127.0.0.1:${global::METRICS_PORT} \ + --authrpc.jwtsecret=${global::HOME}/jwt.hex \ + ${global::RETH_EXTRA_ARGS}`, + shutdown_timeout_secs: 120, + use_blockchain_data: true, + }, + ], + alternative_download: #{ + run_sh: "echo dummy" + }, + upload: #{ + exclude: [ + "jwt.hex", + ] + } +}; + +fn application_status() { + /// "uploading", "downloading", "initializing" statuses are automatically handled by BV if BABEL_CONFIG const is defined + let resp = run_jrpc(#{host: global::API_HOST, method: "/"}); + + if resp.status_code == 200 { + "broadcasting" + } else { + "delinquent" + } +} + + +fn height() { + parse_hex(run_jrpc(#{host: global::API_HOST, method: "eth_blockNumber"}).expect(200).result) +} + + +fn sync_status() { + let resp = run_jrpc(#{host: global::API_HOST, method: "eth_syncing"}).expect(200); + if resp.result == false { + "synced" + } else { + "syncing" + } +} \ No newline at end of file diff --git a/node-base/Dockerfile b/node-base/Dockerfile new file mode 100644 index 00000000..5d8e52a6 --- /dev/null +++ b/node-base/Dockerfile @@ -0,0 +1,83 @@ +FROM debian:bookworm-slim@sha256:a165446a88794db4fec31e35e9441433f9552ae048fb1ed26df352d2b537cb96 + +# Add build arguments for Grafana credentials +ARG GRAFANA_LOKI_BASICAUTH +ARG GRAFANA_PROM_BASICAUTH +ARG GRAFANA_LOKI_API_KEY +ARG GRAFANA_PROM_API_KEY + +ENV DEBIAN_FRONTEND=noninteractive +ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} +ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} +ENV GRAFANA_LOKI_API_KEY=${GRAFANA_LOKI_API_KEY} +ENV GRAFANA_PROM_API_KEY=${GRAFANA_PROM_API_KEY} + +ENV GO_VERSION=1.23.1 +ENV CADVISOR_VERSION=0.49.1 + +RUN apt-get update && apt-get install -y \ + wget \ + curl \ + gnupg \ + debian-keyring \ + debian-archive-keyring \ + apt-transport-https \ + software-properties-common \ + vim \ + psmisc \ + net-tools \ + nano \ + git \ + build-essential \ + ca-certificates \ + libc6 \ + libfontconfig1 \ + apt-utils + +RUN mkdir -p /etc/apt/keyrings/ && \ + wget --inet4-only --no-check-certificate -qO - http://apt.grafana.com/gpg.key | gpg --dearmor -o /etc/apt/keyrings/grafana.gpg && \ + echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] http://apt.grafana.com stable main" > /etc/apt/sources.list.d/grafana.list + +RUN curl -1sLf 'https://dl.cloudsmith.io/public/caddy/xcaddy/gpg.key' | gpg --dearmor -o /usr/share/keyrings/caddy-xcaddy-archive-keyring.gpg && \ + curl -1sLf 'https://dl.cloudsmith.io/public/caddy/xcaddy/debian.deb.txt' > /etc/apt/sources.list.d/caddy-xcaddy.list + +RUN apt-get update && apt-get install -y xcaddy alloy grafana + +RUN wget https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz && \ + tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz && \ + rm go${GO_VERSION}.linux-amd64.tar.gz + +RUN wget https://packages.cloud.google.com/apt/doc/apt-key.gpg -qO /usr/share/keyrings/prometheus-archive-keyring.gpg && \ + curl -fsSL https://ftp-master.debian.org/keys/archive-key-11.asc | gpg --batch --yes --dearmor -o /usr/share/keyrings/debian-archive-keyring-11.gpg && \ + curl -fsSL https://ftp-master.debian.org/keys/archive-key-12.asc | gpg --batch --yes --dearmor -o /usr/share/keyrings/debian-archive-keyring-12.gpg && \ + curl -fsSL https://ftp-master.debian.org/keys/archive-key-12-security.asc | gpg --dearmor -o /usr/share/keyrings/debian-archive-keyring-12-security.gpg + +ENV PATH="/usr/local/go/bin:${PATH}" +ENV GOROOT="/usr/local/go" + +RUN cd / && \ + xcaddy build --with github.com/caddy-dns/cloudflare && \ + mv ./caddy /usr/bin/ && chmod +x /usr/bin/caddy && \ + mkdir -p /etc/caddy + +RUN wget -O /tmp/netdata-kickstart.sh https://my-netdata.io/kickstart.sh && \ + sh /tmp/netdata-kickstart.sh --stable-channel --disable-telemetry --non-interactive --dont-start-it --native-only --disable-cloud + +RUN mkdir -p /var/lib/babel/templates && \ + mkdir -p /var/lib/babel/plugins + +RUN mkdir -p /etc/grafana /var/lib/grafana /usr/share/grafana/conf/provisioning/datasources \ + /usr/share/grafana/conf/provisioning/dashboards /var/lib/grafana/dashboards + +COPY config/grafana-datasource.yml /usr/share/grafana/conf/provisioning/datasources/datasource.yml +COPY config/grafana-dashboard.yml /usr/share/grafana/conf/provisioning/dashboards/dashboard.yml + +#COPY config/caddy-dashboard.json /var/lib/grafana/dashboards + +RUN apt-get autoremove -y && apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +COPY base.rhai /var/lib/babel/plugin/base.rhai +COPY templates/netdata-conf.template /var/lib/babel/templates/netdata-conf.template +COPY templates/netdata-stream-conf.template /var/lib/babel/templates/netdata-stream-conf.template +COPY templates/caddy-dashboard.template /var/lib/babel/templates/caddy-dashboard.template \ No newline at end of file diff --git a/node-base/Dockerfile.ubuntu b/node-base/Dockerfile.ubuntu new file mode 100644 index 00000000..9df6a6b4 --- /dev/null +++ b/node-base/Dockerfile.ubuntu @@ -0,0 +1,51 @@ +FROM ubuntu:22.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + wget \ + curl \ + gnupg \ + debian-keyring \ + debian-archive-keyring \ + apt-transport-https \ + software-properties-common \ + vim \ + psmisc \ + net-tools \ + nano \ + git \ + build-essential + +RUN mkdir -p /etc/apt/keyrings/ && \ + wget --inet4-only --no-check-certificate -qO - http://apt.grafana.com/gpg.key | gpg --dearmor -o /etc/apt/keyrings/grafana.gpg && \ + echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] http://apt.grafana.com stable main" > /etc/apt/sources.list.d/grafana.list + +RUN curl -1sLf 'https://dl.cloudsmith.io/public/caddy/xcaddy/gpg.key' | gpg --dearmor -o /usr/share/keyrings/caddy-xcaddy-archive-keyring.gpg && \ + curl -1sLf 'https://dl.cloudsmith.io/public/caddy/xcaddy/debian.deb.txt' > /etc/apt/sources.list.d/caddy-xcaddy.list + +RUN add-apt-repository -y universe && apt-get update + +RUN apt-get install -y alloy vim psmisc xcaddy + +RUN wget https://go.dev/dl/go1.23.1.linux-amd64.tar.gz && \ + tar -C /usr/local -xzf go1.23.1.linux-amd64.tar.gz && \ + rm go1.23.1.linux-amd64.tar.gz + +ENV PATH="/usr/local/go/bin:${PATH}" +ENV GOROOT="/usr/local/go" + +RUN cd / && \ + xcaddy build --with github.com/caddy-dns/cloudflare && \ + mv ./caddy /usr/bin/ && chmod +x /usr/bin/caddy && \ + mkdir -p /etc/caddy + +RUN wget -O /tmp/netdata-kickstart.sh https://my-netdata.io/kickstart.sh && \ + sh /tmp/netdata-kickstart.sh --stable-channel --disable-telemetry --non-interactive --dont-start-it --native-only --disable-cloud + +RUN mkdir -p /var/lib/babel/templates && \ + mkdir -p /var/lib/babel/plugins + +COPY base.rhai /var/lib/babel/plugin/base.rhai +COPY netdata-conf.template /var/lib/babel/templates/netdata-conf.template +COPY netdata-stream-conf.template /var/lib/babel/templates/netdata-stream-conf.template diff --git a/node-base/base.rhai b/node-base/base.rhai new file mode 100644 index 00000000..60f9df3b --- /dev/null +++ b/node-base/base.rhai @@ -0,0 +1,31 @@ +export const BASE_CONFIG = #{ + config_files: [ + #{ + template: "/var/lib/babel/templates/netdata-conf.template", + destination: "/etc/netdata/netdata.conf", + params: #{} + }, + #{ + template: "/var/lib/babel/templates/netdata-stream-conf.template", + destination: "/etc/netdata/stream.conf", + params: #{} + }, + #{ + template: "/var/lib/babel/templates/caddy-dashboard.template", + destination: "/var/lib/grafana/dashboards/caddy-dashboard.json", + params: #{ + node_name: node_env().node_name + }, + }, + ], + services : [ + #{ + name: "grafana", + run_sh: "GF_USERS_DEFAULT_ROLE=Viewer \ + GF_AUTH_ANONYMOUS_ENABLED=true \ + GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer \ + GF_ROLE_VIEWER_EDITORS_CAN_SAVE=false \ + grafana-server --homepath /usr/share/grafana --config /etc/grafana/grafana.ini" + }, + ] +} diff --git a/node-base/config/caddy-dashboard.json b/node-base/config/caddy-dashboard.json new file mode 100644 index 00000000..0292ece2 --- /dev/null +++ b/node-base/config/caddy-dashboard.json @@ -0,0 +1,2361 @@ +{ + "annotations": { + "list": [ + { + "$$hashKey": "object:1058", + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Monitoring Caddy Server with Grafana (Prometheus + Loki)", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 9, + "panels": [], + "title": "Synthesis", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P520024444597A5C0" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "srv0" + }, + "properties": [ + { + "id": "displayName", + "value": "Port: 80" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "srv1" + }, + "properties": [ + { + "id": "displayName", + "value": "Port: 443" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 0, + "y": 1 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "caddy_http_requests_total{instance=~\"$host\",handler=~\"$handler\"}", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{server}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of requests processed", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P520024444597A5C0" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 5, + "y": 1 + }, + "id": 21, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 50 + }, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "rate(caddy_http_requests_total{instance=~\"$host\",handler=~\"$handler\"}[5m])", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Rate of requests per second (avg 5m)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P520024444597A5C0" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 10, + "y": 1 + }, + "id": 24, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 50 + }, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "sum(rate(caddy_http_request_duration_seconds_count{instance=~\"$host\",handler=~\"$handler\"}[5m])) by (handler)", + "format": "heatmap", + "hide": false, + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Average response time", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P520024444597A5C0" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-red", + "mode": "fixed" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 15, + "y": 1 + }, + "id": 19, + "maxDataPoints": 10000, + "options": { + "colorMode": "background_solid", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "diff" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 50 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "sum(caddy_http_request_duration_seconds_count{instance=~\"$host\",handler=~\"$handler\"}) by (code)", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "refId": "A" + } + ], + "title": "HTTP Status Errors", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "include": [ + "400", + "401", + "404", + "410", + "500" + ], + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-red", + "mode": "fixed" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 23, + "options": { + "colorMode": "background_solid", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "diff" + ], + "fields": "", + "values": false + }, + "showPercentChange": true, + "text": { + "valueSize": 50 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "sum(caddy_http_request_errors_total{instance=~\"$host\",handler=~\"$handler\"}) by (handler)", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "refId": "A" + } + ], + "title": "Middleware Errors", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "include": [ + "400", + "401", + "404", + "410", + "500" + ], + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 8, + "panels": [], + "title": "API REST Caddy", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 0, + "y": 7 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "expr": "avg(rate(caddy_http_requests_total{instance=~\"$host\",handler=~\"$handler\"}[5m])*10) by (handler)", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "refId": "A" + } + ], + "title": "HTTP(S) requests total", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 5, + "y": 7 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "avg(avg_over_time(caddy_http_requests_in_flight{instance=~\"$host\",handler=~\"$handler\"}[1m])) by (handler)", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "refId": "A" + } + ], + "title": "Requests In Flight", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 14, + "x": 10, + "y": 7 + }, + "id": 22, + "maxDataPoints": 100, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + }, + "yBuckets": { + "mode": "size", + "scale": { + "type": "linear" + }, + "value": "" + } + }, + "cellGap": 2, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Magma", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "sum(increase(caddy_http_request_duration_seconds_bucket{instance=~\"$host\", handler=~\"$handler\"}[5m])) by (le)", + "format": "heatmap", + "hide": false, + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Request Duration (heatmap)", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "Median response duration in seconds", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 0, + "y": 12 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(caddy_http_response_duration_seconds_bucket[5m])))", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Median response duration in seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 5, + "y": 12 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "expr": "sum(rate(caddy_http_request_duration_seconds_count{instance=~\"$host\",handler=~\"$handler\"}[5m])) by (handler)", + "interval": "", + "legendFormat": "{{handler}}", + "refId": "A" + } + ], + "title": "Request/Response duration seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "displayName": "Median response size in bytes", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 0, + "y": 17 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(caddy_http_response_size_bytes_bucket{instance=~\"$host\"}[5m])))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Median response size in bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 5, + "y": 17 + }, + "id": 17, + "maxDataPoints": 10000, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "sum(irate(caddy_http_request_duration_seconds_count{instance=~\"$host\",handler=~\"$handler\"}[1m])) by (code)", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "refId": "A" + } + ], + "title": "Requests by Response Code", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 10, + "x": 0, + "y": 22 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(caddy_http_request_duration_seconds_bucket{instance=~\"$host\",handler=~\"$handler\"}[1m])) by (le))", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(caddy_http_request_duration_seconds_bucket{instance=~\"$host\",handler=~\"$handler\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(caddy_http_request_duration_seconds_bucket{instance=~\"$host\",handler=~\"$handler\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.75, sum(rate(caddy_http_request_duration_seconds_bucket{instance=~\"$host\",handler=~\"$handler\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(caddy_http_request_duration_seconds_bucket{instance=~\"$host\",handler=~\"$handler\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "refId": "E" + } + ], + "title": "Request Duration (percentile)", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "Time", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "histogram_quantile(0.5, sum(rate(caddy_http_request_duration_seconds_bucket{instance=~\".*\",handler=~\".*\"}[1m])) by (le))": "p50", + "histogram_quantile(0.75, sum(rate(caddy_http_request_duration_seconds_bucket{instance=~\".*\",handler=~\".*\"}[1m])) by (le))": "p75", + "histogram_quantile(0.90, sum(rate(caddy_http_request_duration_seconds_bucket{instance=~\".*\",handler=~\".*\"}[1m])) by (le))": "p90", + "histogram_quantile(0.95, sum(rate(caddy_http_request_duration_seconds_bucket{instance=~\".*\",handler=~\".*\"}[1m])) by (le))": "p95", + "histogram_quantile(0.99, sum(rate(caddy_http_request_duration_seconds_bucket{instance=~\".*\",handler=~\".*\"}[1m])) by (le))": "p99" + } + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "fieldMinMax": false, + "mappings": [], + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^([2-2]\\d*)$" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#aaffcab0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^([3-3]\\d*)$" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^([4-4]\\d*)$" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^([5-5]\\d*)$" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 10, + "y": 22 + }, + "id": 18, + "maxDataPoints": 10000, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "diff" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "sum(caddy_http_request_duration_seconds_count{instance=~\"$host\",handler=~\"$handler\"}) by (code)", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "refId": "A" + } + ], + "title": "HTTP Status", + "transparent": true, + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-red", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 22 + }, + "id": 29, + "maxDataPoints": 10000, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(code) (caddy_http_request_duration_seconds_count{instance=~\"$host\", handler=~\"$handler\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{code}}", + "refId": "A", + "useBackend": false + } + ], + "title": "HTTP Status Errors", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "include": [ + "400", + "401", + "404", + "410", + "500" + ], + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-red", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 27 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "avg(rate(caddy_http_request_errors_total{instance=~\"$host\",handler=~\"$handler\"}[5m])*10) by (handler)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "range": true, + "refId": "A" + } + ], + "title": "Middleware Errors", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 7, + "panels": [], + "title": "Logs Caddy", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "grafanacloud-logs" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 287 + } + ] + } + ] + }, + "gridPos": { + "h": 17, + "w": 18, + "x": 0, + "y": 32 + }, + "id": 26, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "detected_level" + } + ] + }, + "pluginVersion": "11.4.0-78571", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "grafanacloud-logs" + }, + "editorMode": "code", + "expr": "{filename=\"$logfile\", instance=~\"$host\"} | pattern `<_>|` | line_format `{{.message}}` | json | line_format \"{{.status}}\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Rag logs in table", + "transformations": [ + { + "id": "extractFields", + "options": { + "format": "json", + "jsonPaths": [], + "keepTime": true, + "replace": true, + "source": "labels" + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "grafanacloud-logs" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "fieldMinMax": false, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Client IP" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "Geo IP", + "url": "https://iplocation.io/ip/${__data.fields[\"Client IP\"]}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 202 + } + ] + } + ] + }, + "gridPos": { + "h": 17, + "w": 6, + "x": 18, + "y": 32 + }, + "id": 28, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Time" + } + ] + }, + "pluginVersion": "11.4.0-78571", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "grafanacloud-logs" + }, + "editorMode": "builder", + "expr": "rate({filename=\"$logfile\"} | json | line_format `{{.status}}` [$__auto])", + "queryType": "range", + "refId": "A" + } + ], + "title": "Client IP", + "transformations": [ + { + "id": "extractFields", + "options": { + "format": "json", + "jsonPaths": [], + "keepTime": true, + "replace": true, + "source": "labels" + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "Status Count": { + "aggregations": [ + "count" + ], + "operation": "aggregate" + }, + "Time": { + "aggregations": [], + "operation": "groupby" + }, + "request_client_ip": { + "aggregations": [], + "operation": "groupby" + }, + "status": { + "aggregations": [] + } + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "status\\status": true + }, + "includeByName": {}, + "indexByName": { + "200": 6, + "301": 5, + "302": 4, + "304": 3, + "404": 2, + "410": 1, + "500": 0, + "status\\status": 7 + }, + "renameByName": { + "Time": "", + "request_client_ip": "Client IP" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "grafanacloud-logs" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 33, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 25, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": true, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.4.0-78029", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "grafanacloud-logs" + }, + "editorMode": "code", + "expr": "{filename=\"$logfile\",instance=~\"$host\"} | pattern `<_>|` | line_format `{{.message}}` | json | line_format \"- [HTTP Status: {{.status}} - Duration: {{.duration}} - Client IP: {{.request_client_ip}} - Request: {{.request_uri}}]\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Raw logs", + "type": "logs" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [ + "Caddy" + ], + "templating": { + "list": [ + { + "current": { + "text": "grafanacloud-blockjoy-logs", + "value": "grafanacloud-logs" + }, + "hide": 2, + "includeAll": false, + "label": "Datasource Loki", + "name": "datasourceloki", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "/grafanacloud-blockjoy-logs/", + "type": "datasource" + }, + { + "current": { + "text": "/var/lib/babel/jobs/logs/caddy", + "value": "/var/lib/babel/jobs/logs/caddy" + }, + "datasource": { + "type": "loki", + "uid": "PA2B326B1CBC5D7D2" + }, + "definition": "", + "includeAll": false, + "label": "Log file", + "name": "logfile", + "options": [], + "query": { + "label": "filename", + "refId": "LokiVariableQueryEditor-VariableQuery", + "stream": "{instance=~\"$host\"}", + "type": 1 + }, + "refresh": 1, + "regex": "/caddy/", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "grafanacloud-blockjoy-prom", + "value": "grafanacloud-prom" + }, + "hide": 2, + "includeAll": false, + "label": "Datasource Prometheus", + "name": "datasourceprometheus", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "/grafanacloud-blockjoy-prom/", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "annually-whole-caribou.n0des.xyz", + "value": "annually-whole-caribou.n0des.xyz" + }, + "datasource": { + "type": "prometheus", + "uid": "P520024444597A5C0" + }, + "definition": "label_values(up{job=~\"caddy\"},instance)", + "includeAll": false, + "label": "Host", + "name": "host", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(up{job=~\"caddy\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "P520024444597A5C0" + }, + "definition": "label_values({__name__=~\"^caddy.*\", handler=~\".*\"},handler)", + "includeAll": true, + "label": "Handler", + "name": "handler", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({__name__=~\"^caddy.*\", handler=~\".*\"},handler)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Caddy Monitoring", + "uid": "bdh0qkohvn5s0a", + "version": 3, + "weekStart": "" +} diff --git a/node-base/config/grafana-dashboard.yml b/node-base/config/grafana-dashboard.yml new file mode 100644 index 00000000..e0a52503 --- /dev/null +++ b/node-base/config/grafana-dashboard.yml @@ -0,0 +1,12 @@ +apiVersion: 1 +providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + updateIntervalSeconds: 10 + options: + path: /var/lib/grafana/dashboards + dashboards: + - file: /var/lib/grafana/dashboards/caddy-dashboard.json diff --git a/node-base/config/grafana-datasource.yml b/node-base/config/grafana-datasource.yml new file mode 100644 index 00000000..78605a4f --- /dev/null +++ b/node-base/config/grafana-datasource.yml @@ -0,0 +1,20 @@ +apiVersion: 1 +datasources: + - name: grafanacloud-prom + type: prometheus + url: https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom + access: server + isDefault: true + basicAuth: true + basicAuthUser: 1322871 + secureJsonData: + basicAuthPassword: ${GRAFANA_PROM_API_KEY} + - name: grafanacloud-logs + type: loki + url: https://logs-prod-012.grafana.net + access: server + isDefault: false + basicAuth: true + basicAuthUser: 760471 + secureJsonData: + basicAuthPassword: ${GRAFANA_LOKI_API_KEY} diff --git a/node-base/templates/caddy-dashboard.template b/node-base/templates/caddy-dashboard.template new file mode 100644 index 00000000..a3a2cd8d --- /dev/null +++ b/node-base/templates/caddy-dashboard.template @@ -0,0 +1,2366 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "$$hashKey": "object:1058", + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Monitoring Caddy Server with Grafana (Prometheus + Loki)", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 9, + "panels": [], + "title": "Synthesis", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "srv0" + }, + "properties": [ + { + "id": "displayName", + "value": "Port: 80" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "srv1" + }, + "properties": [ + { + "id": "displayName", + "value": "Port: 443" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 0, + "y": 1 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "caddy_http_requests_total{instance=~\"$host\",handler=~\"$handler\"}", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{server}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of requests processed", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 5, + "y": 1 + }, + "id": 21, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 50 + }, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "rate(caddy_http_requests_total{instance=~\"$host\",handler=~\"$handler\"}[5m])", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Rate of requests per second (avg 5m)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 10, + "y": 1 + }, + "id": 24, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 50 + }, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "sum(rate(caddy_http_request_duration_seconds_count{instance=~\"$host\",handler=~\"$handler\"}[5m])) by (handler)", + "format": "heatmap", + "hide": false, + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Average response time", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-red", + "mode": "fixed" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 15, + "y": 1 + }, + "id": 19, + "maxDataPoints": 10000, + "options": { + "colorMode": "background_solid", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "diff" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 50 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "sum(caddy_http_request_duration_seconds_count{instance=~\"$host\",handler=~\"$handler\"}) by (code)", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "refId": "A" + } + ], + "title": "HTTP Status Errors", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "include": [ + "400", + "401", + "404", + "410", + "500" + ], + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-red", + "mode": "fixed" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 23, + "options": { + "colorMode": "background_solid", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "diff" + ], + "fields": "", + "values": false + }, + "showPercentChange": true, + "text": { + "valueSize": 50 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "sum(caddy_http_request_errors_total{instance=~\"$host\",handler=~\"$handler\"}) by (handler)", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "refId": "A" + } + ], + "title": "Middleware Errors", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "include": [ + "400", + "401", + "404", + "410", + "500" + ], + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 8, + "panels": [], + "title": "API REST Caddy", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 0, + "y": 7 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "expr": "avg(rate(caddy_http_requests_total{instance=~\"$host\",handler=~\"$handler\"}[5m])*10) by (handler)", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "refId": "A" + } + ], + "title": "HTTP(S) requests total", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 5, + "y": 7 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "avg(avg_over_time(caddy_http_requests_in_flight{instance=~\"$host\",handler=~\"$handler\"}[1m])) by (handler)", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "refId": "A" + } + ], + "title": "Requests In Flight", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 14, + "x": 10, + "y": 7 + }, + "id": 22, + "maxDataPoints": 100, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + }, + "yBuckets": { + "mode": "size", + "scale": { + "type": "linear" + }, + "value": "" + } + }, + "cellGap": 2, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Magma", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "sum(increase(caddy_http_request_duration_seconds_bucket{instance=~\"$host\", handler=~\"$handler\"}[5m])) by (le)", + "format": "heatmap", + "hide": false, + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Request Duration (heatmap)", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "Median response duration in seconds", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 0, + "y": 12 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(caddy_http_response_duration_seconds_bucket[5m])))", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Median response duration in seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 5, + "y": 12 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "expr": "sum(rate(caddy_http_request_duration_seconds_count{instance=~\"$host\",handler=~\"$handler\"}[5m])) by (handler)", + "interval": "", + "legendFormat": "{{handler}}", + "refId": "A" + } + ], + "title": "Request/Response duration seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "displayName": "Median response size in bytes", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 0, + "y": 17 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(caddy_http_response_size_bytes_bucket{instance=~\"$host\"}[5m])))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Median response size in bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 5, + "y": 17 + }, + "id": 17, + "maxDataPoints": 10000, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "sum(irate(caddy_http_request_duration_seconds_count{instance=~\"$host\",handler=~\"$handler\"}[1m])) by (code)", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "refId": "A" + } + ], + "title": "Requests by Response Code", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 10, + "x": 0, + "y": 22 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(caddy_http_request_duration_seconds_bucket{instance=~\"$host\",handler=~\"$handler\"}[1m])) by (le))", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(caddy_http_request_duration_seconds_bucket{instance=~\"$host\",handler=~\"$handler\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(caddy_http_request_duration_seconds_bucket{instance=~\"$host\",handler=~\"$handler\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.75, sum(rate(caddy_http_request_duration_seconds_bucket{instance=~\"$host\",handler=~\"$handler\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(caddy_http_request_duration_seconds_bucket{instance=~\"$host\",handler=~\"$handler\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "refId": "E" + } + ], + "title": "Request Duration (percentile)", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "Time", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "histogram_quantile(0.5, sum(rate(caddy_http_request_duration_seconds_bucket{instance=~\".*\",handler=~\".*\"}[1m])) by (le))": "p50", + "histogram_quantile(0.75, sum(rate(caddy_http_request_duration_seconds_bucket{instance=~\".*\",handler=~\".*\"}[1m])) by (le))": "p75", + "histogram_quantile(0.90, sum(rate(caddy_http_request_duration_seconds_bucket{instance=~\".*\",handler=~\".*\"}[1m])) by (le))": "p90", + "histogram_quantile(0.95, sum(rate(caddy_http_request_duration_seconds_bucket{instance=~\".*\",handler=~\".*\"}[1m])) by (le))": "p95", + "histogram_quantile(0.99, sum(rate(caddy_http_request_duration_seconds_bucket{instance=~\".*\",handler=~\".*\"}[1m])) by (le))": "p99" + } + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "fieldMinMax": false, + "mappings": [], + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^([2-2]\\d*)$" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#aaffcab0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^([3-3]\\d*)$" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^([4-4]\\d*)$" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^([5-5]\\d*)$" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 10, + "y": 22 + }, + "id": 18, + "maxDataPoints": 10000, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "diff" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "sum(caddy_http_request_duration_seconds_count{instance=~\"$host\",handler=~\"$handler\"}) by (code)", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "refId": "A" + } + ], + "title": "HTTP Status", + "transparent": true, + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-red", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 22 + }, + "id": 29, + "maxDataPoints": 10000, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(code) (caddy_http_request_duration_seconds_count{instance=~\"$host\", handler=~\"$handler\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{code}}", + "refId": "A", + "useBackend": false + } + ], + "title": "HTTP Status Errors", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "include": [ + "400", + "401", + "404", + "410", + "500" + ], + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-red", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 27 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "avg(rate(caddy_http_request_errors_total{instance=~\"$host\",handler=~\"$handler\"}[5m])*10) by (handler)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "range": true, + "refId": "A" + } + ], + "title": "Middleware Errors", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 7, + "panels": [], + "title": "Logs Caddy", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "grafanacloud-logs" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 287 + } + ] + } + ] + }, + "gridPos": { + "h": 17, + "w": 18, + "x": 0, + "y": 32 + }, + "id": 26, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "detected_level" + } + ] + }, + "pluginVersion": "11.4.0-78571", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "grafanacloud-logs" + }, + "editorMode": "code", + "expr": "{filename=\"$logfile\", instance=~\"$host\"} | pattern `<_>|` | line_format `{{.message}}` | json | line_format \"{{.status}}\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Rag logs in table", + "transformations": [ + { + "id": "extractFields", + "options": { + "format": "json", + "jsonPaths": [], + "keepTime": true, + "replace": true, + "source": "labels" + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "grafanacloud-logs" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "fieldMinMax": false, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Client IP" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "Geo IP", + "url": "https://iplocation.io/ip/${__data.fields[\"Client IP\"]}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 202 + } + ] + } + ] + }, + "gridPos": { + "h": 17, + "w": 6, + "x": 18, + "y": 32 + }, + "id": 28, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Time" + } + ] + }, + "pluginVersion": "11.4.0-78571", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "grafanacloud-logs" + }, + "editorMode": "builder", + "expr": "rate({filename=\"$logfile\"} | json | line_format `{{.status}}` [$__auto])", + "queryType": "range", + "refId": "A" + } + ], + "title": "Client IP", + "transformations": [ + { + "id": "extractFields", + "options": { + "format": "json", + "jsonPaths": [], + "keepTime": true, + "replace": true, + "source": "labels" + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "Status Count": { + "aggregations": [ + "count" + ], + "operation": "aggregate" + }, + "Time": { + "aggregations": [], + "operation": "groupby" + }, + "request_client_ip": { + "aggregations": [], + "operation": "groupby" + }, + "status": { + "aggregations": [] + } + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "status\\status": true + }, + "includeByName": {}, + "indexByName": { + "200": 6, + "301": 5, + "302": 4, + "304": 3, + "404": 2, + "410": 1, + "500": 0, + "status\\status": 7 + }, + "renameByName": { + "Time": "", + "request_client_ip": "Client IP" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "grafanacloud-logs" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 33, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 25, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": true, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.4.0-78029", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "grafanacloud-logs" + }, + "editorMode": "code", + "expr": "{filename=\"$logfile\",instance=~\"$host\"} | pattern `<_>|` | line_format `{{.message}}` | json | line_format \"- [HTTP Status: {{.status}} - Duration: {{.duration}} - Client IP: {{.request_client_ip}} - Request: {{.request_uri}}]\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Raw logs", + "type": "logs" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [ + "Caddy" + ], + "templating": { + "list": [ + { + "current": { + "text": "grafanacloud-blockjoy-logs", + "value": "grafanacloud-logs" + }, + "hide": 2, + "includeAll": false, + "label": "Datasource Loki", + "name": "datasourceloki", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "/grafanacloud-blockjoy-logs/", + "type": "datasource" + }, + { + "current": { + "text": "/var/lib/babel/jobs/logs/caddy", + "value": "/var/lib/babel/jobs/logs/caddy" + }, + "datasource": { + "type": "loki", + "uid": "grafanacloud-logs" + }, + "definition": "", + "includeAll": false, + "label": "Log file", + "name": "logfile", + "options": [], + "query": { + "label": "filename", + "refId": "LokiVariableQueryEditor-VariableQuery", + "stream": "{instance=~\"$host\"}", + "type": 1 + }, + "refresh": 1, + "regex": "/caddy/", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "grafanacloud-blockjoy-prom", + "value": "grafanacloud-prom" + }, + "hide": 2, + "includeAll": false, + "label": "Datasource Prometheus", + "name": "datasourceprometheus", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "/grafanacloud-blockjoy-prom/", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { +{% endraw %} + "text": "{{ node_name }}.n0des.xyz", + "value": "{{ node_name }}.n0des.xyz" +{% raw %} + }, + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "definition": "label_values(up{job=~\"caddy\"},instance)", + "hide": 2, + "includeAll": false, + "label": "Host", + "name": "host", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(up{job=~\"caddy\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "definition": "label_values({__name__=~\"^caddy.*\", handler=~\".*\"},handler)", + "includeAll": true, + "label": "Handler", + "name": "handler", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({__name__=~\"^caddy.*\", handler=~\".*\"},handler)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Caddy Monitoring", + "uid": "bdh0qkohvn5s0a", + "version": 3, + "weekStart": "" +} +{% endraw %} diff --git a/node-base/templates/netdata-conf.template b/node-base/templates/netdata-conf.template new file mode 100644 index 00000000..85d1a726 --- /dev/null +++ b/node-base/templates/netdata-conf.template @@ -0,0 +1,9 @@ +[global] + run as user = root +[web] + mode = none +[db] + mode = ram + update every = 5 +[health] + enabled = no diff --git a/node-base/templates/netdata-stream-conf.template b/node-base/templates/netdata-stream-conf.template new file mode 100644 index 00000000..5375ef78 --- /dev/null +++ b/node-base/templates/netdata-stream-conf.template @@ -0,0 +1,10 @@ +[stream] + enabled = yes + destination = tcp:netdata.blkjy.io + api key = 894aa814-e672-4df1-9e8c-3bc919f1b905 + timeout seconds = 60 + default port = 19996 + send charts matching = * + buffer size bytes = 10485760 + reconnect delay seconds = 5 + initial clock resync iterations = 60 diff --git a/renovate.json b/renovate.json new file mode 100644 index 00000000..1574bcfa --- /dev/null +++ b/renovate.json @@ -0,0 +1,52 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:base", + "docker:enableMajor" + ], + "enabledManagers": ["dockerfile", "github-actions"], + "dependencyDashboard": true, + "pinDigests": true, + "repositories": ["blockjoy/blockjoy-protocols"], + "packageRules": [ + { + "matchDatasources": ["docker"], + "matchUpdateTypes": ["major", "minor", "patch", "digest"], + "enabled": true + }, + { + "description": "Base image updates", + "matchPaths": ["node-base/Dockerfile"], + "groupName": "base-dependencies", + "separateMinorPatch": true + }, + { + "description": "Client image updates", + "matchPaths": ["clients/**/Dockerfile"], + "matchPackagePatterns": ["ghcr.io/blockjoy/node-base"], + "automerge": true, + "automergeType": "branch", + "groupName": "client-dependencies" + }, + { + "description": "Protocol implementation updates", + "matchPaths": ["ethereum/**/Dockerfile"], + "matchPackagePatterns": ["ghcr.io/blockjoy/blockjoy-*"], + "automerge": true, + "automergeType": "branch", + "groupName": "protocol-dependencies" + } + ], + "regexManagers": [ + { + "fileMatch": ["^.+/Dockerfile$"], + "matchStrings": [ + "FROM ghcr.io/blockjoy/(?[^:]+):(?(?:[0-9]+\\.[0-9]+\\.[0-9]+-)?[a-f0-9]+)" + ], + "datasourceTemplate": "docker", + "registryUrlTemplate": "https://ghcr.io" + } + ], + "prHourlyLimit": 0, + "prConcurrentLimit": 0 +} From 8238e3a872a40fcc32adaee0a9fe111f2310afc9 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 25 Nov 2024 16:09:40 +0200 Subject: [PATCH 02/88] Fixed --- .github/workflows/docker-build-test.yml | 83 +++++++++++++++++++++ .github/workflows/docker-publish.yml | 95 +++++++++++++++++++++++++ clients/erigon/Caddyfile.template | 2 +- clients/erigon/Dockerfile | 4 +- clients/erigon/config-alloy.template | 4 +- clients/reth/Caddyfile.template | 2 +- clients/reth/Dockerfile | 19 ++--- clients/reth/config-alloy.template | 2 +- node-base/Dockerfile | 5 -- node-base/config/grafana-datasource.yml | 4 +- 10 files changed, 198 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/docker-build-test.yml create mode 100644 .github/workflows/docker-publish.yml diff --git a/.github/workflows/docker-build-test.yml b/.github/workflows/docker-build-test.yml new file mode 100644 index 00000000..b411166c --- /dev/null +++ b/.github/workflows/docker-build-test.yml @@ -0,0 +1,83 @@ +name: Docker Build Test + +on: + pull_request: + branches: [ main ] + paths: + - '**/Dockerfile' + +# Configure GITHUB_TOKEN permissions +permissions: + contents: read + packages: read + pull-requests: read + id-token: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test-build: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + + - name: Generate build matrix + id: set-matrix + run: | + DIRS=$(find . -name Dockerfile -exec dirname {} \; | sed 's|^\./||' | jq -R -s -c 'split("\n")[:-1]') + echo "matrix=${DIRS}" >> $GITHUB_OUTPUT + + build-test: + needs: test-build + runs-on: ubuntu-latest + strategy: + matrix: + image_path: ${{ fromJson(needs.test-build.outputs.matrix) }} + steps: + - uses: actions/checkout@v4 + + - name: Generate version + id: version + run: | + if [[ "${{ matrix.image_path }}" == clients/* ]]; then + IMAGE_NAME="blockjoy-$(basename ${{ matrix.image_path }})" + if [[ -f "${{ matrix.image_path }}/Dockerfile" ]]; then + CLIENT_VERSION=$(grep -E "ENV.*VERSION[[:space:]]*[=][[:space:]]*v?[0-9]+\.[0-9]+\.[0-9]+" "${{ matrix.image_path }}/Dockerfile" | grep -oE "v?[0-9]+\.[0-9]+\.[0-9]+") + if [[ ! -z "$CLIENT_VERSION" ]]; then + CLIENT_VERSION=${CLIENT_VERSION#v} + IMAGE_TAG="${CLIENT_VERSION}-$(git rev-parse --short HEAD)" + else + IMAGE_TAG="$(git rev-parse --short HEAD)" + fi + else + IMAGE_TAG="$(git rev-parse --short HEAD)" + fi + else + IMAGE_NAME=$(echo ${{ matrix.image_path }} | tr '/' '-') + IMAGE_TAG="$(git rev-parse --short HEAD)" + fi + echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT + echo "image_tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Test build image + uses: docker/build-push-action@v5 + with: + context: ./${{ matrix.image_path }} + push: false + load: true + build-args: | + ${{ startsWith(matrix.image_path, 'node-base') && format('GRAFANA_LOKI_API_KEY={0} + GRAFANA_PROM_API_KEY={1}', secrets.GRAFANA_LOKI_API_KEY, secrets.GRAFANA_PROM_API_KEY) || '' }} + ${{ startsWith(matrix.image_path, 'clients') && format('GRAFANA_LOKI_BASICAUTH={0} + GRAFANA_PROM_BASICAUTH={1} + CLOUDFLARE_API_KEY={2}', secrets.GRAFANA_LOKI_BASICAUTH, secrets.GRAFANA_PROM_BASICAUTH, secrets.CLOUDFLARE_API_KEY) || '' }} + cache-from: type=gha + cache-to: type=gha,mode=max + tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 00000000..14222040 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,95 @@ +name: Docker Build and Publish + +on: + push: + branches: [ main ] + paths: + - '**/Dockerfile' + workflow_dispatch: + +# Configure GITHUB_TOKEN permissions +permissions: + contents: read + packages: write + pull-requests: write + id-token: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-and-publish: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + + - name: Generate build matrix + id: set-matrix + run: | + DIRS=$(find . -name Dockerfile -exec dirname {} \; | sed 's|^\./||' | jq -R -s -c 'split("\n")[:-1]') + echo "matrix=${DIRS}" >> $GITHUB_OUTPUT + + publish: + needs: build-and-publish + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + pull-requests: write + id-token: write + strategy: + matrix: + image_path: ${{ fromJson(needs.build-and-publish.outputs.matrix) }} + steps: + - uses: actions/checkout@v4 + + - name: Generate version + id: version + run: | + if [[ "${{ matrix.image_path }}" == clients/* ]]; then + IMAGE_NAME="blockjoy-$(basename ${{ matrix.image_path }})" + if [[ -f "${{ matrix.image_path }}/Dockerfile" ]]; then + CLIENT_VERSION=$(grep -E "ENV.*VERSION[[:space:]]*[=][[:space:]]*v?[0-9]+\.[0-9]+\.[0-9]+" "${{ matrix.image_path }}/Dockerfile" | grep -oE "v?[0-9]+\.[0-9]+\.[0-9]+") + if [[ ! -z "$CLIENT_VERSION" ]]; then + CLIENT_VERSION=${CLIENT_VERSION#v} + IMAGE_TAG="${CLIENT_VERSION}-$(git rev-parse --short HEAD)" + else + IMAGE_TAG="$(git rev-parse --short HEAD)" + fi + else + IMAGE_TAG="$(git rev-parse --short HEAD)" + fi + else + IMAGE_NAME=$(echo ${{ matrix.image_path }} | tr '/' '-') + IMAGE_TAG="$(git rev-parse --short HEAD)" + fi + echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT + echo "image_tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push image + uses: docker/build-push-action@v5 + with: + context: ./${{ matrix.image_path }} + push: true + build-args: | + ${{ startsWith(matrix.image_path, 'node-base') && format('GRAFANA_LOKI_API_KEY={0} + GRAFANA_PROM_API_KEY={1}', secrets.GRAFANA_LOKI_API_KEY, secrets.GRAFANA_PROM_API_KEY) || '' }} + ${{ startsWith(matrix.image_path, 'clients') && format('GRAFANA_LOKI_BASICAUTH={0} + GRAFANA_PROM_BASICAUTH={1} + CLOUDFLARE_API_KEY={2}', secrets.GRAFANA_LOKI_BASICAUTH, secrets.GRAFANA_PROM_BASICAUTH, secrets.CLOUDFLARE_API_KEY) || '' }} + cache-from: type=gha + cache-to: type=gha,mode=max + tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} diff --git a/clients/erigon/Caddyfile.template b/clients/erigon/Caddyfile.template index f5bc8a26..21e9d359 100644 --- a/clients/erigon/Caddyfile.template +++ b/clients/erigon/Caddyfile.template @@ -6,7 +6,7 @@ format json level INFO } - acme_dns cloudflare UR8KSq7bqAWNJkpol5eEQPiU_K0ADrdXzCZoXEl3 + acme_dns cloudflare {$CLOUDFLARE_API_KEY} auto_https disable_redirects storage file_system {{ data_dir }} email ops@blockjoy.com diff --git a/clients/erigon/Dockerfile b/clients/erigon/Dockerfile index 756dc23d..55aed017 100644 --- a/clients/erigon/Dockerfile +++ b/clients/erigon/Dockerfile @@ -1,10 +1,12 @@ FROM ghcr.io/blockjoy/node-base:abc123 -# Add build arguments for Grafana credentials +# Add build arguments for Cloudflare and Grafana push credentials +ARG CLOUDFLARE_API_KEY ARG GRAFANA_LOKI_BASICAUTH ARG GRAFANA_PROM_BASICAUTH ENV DEBIAN_FRONTEND=noninteractive +ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} diff --git a/clients/erigon/config-alloy.template b/clients/erigon/config-alloy.template index 0734ac5d..9009bf13 100644 --- a/clients/erigon/config-alloy.template +++ b/clients/erigon/config-alloy.template @@ -117,7 +117,7 @@ loki.write "logs_default" { basic_auth { username = "760471" - password = ${GRAFANA_LOKI_BASICAUTH} + password = "${GRAFANA_LOKI_BASICAUTH}" } } external_labels = {} @@ -196,7 +196,7 @@ prometheus.remote_write "default" { basic_auth { username = "1322871" - password = ${GRAFANA_PROM_BASICAUTH} + password = "${GRAFANA_PROM_BASICAUTH}" } queue_config { } diff --git a/clients/reth/Caddyfile.template b/clients/reth/Caddyfile.template index 91fb841f..aeedb66c 100644 --- a/clients/reth/Caddyfile.template +++ b/clients/reth/Caddyfile.template @@ -6,7 +6,7 @@ format json level INFO } - acme_dns cloudflare UR8KSq7bqAWNJkpol5eEQPiU_K0ADrdXzCZoXEl3 + acme_dns cloudflare {$CLOUDFLARE_API_KEY} auto_https disable_redirects storage file_system {{ data_dir }} email ops@blockjoy.com diff --git a/clients/reth/Dockerfile b/clients/reth/Dockerfile index c64e998c..e5f92c0e 100644 --- a/clients/reth/Dockerfile +++ b/clients/reth/Dockerfile @@ -1,15 +1,16 @@ -FROM node-base:0.0.1 +FROM ghcr.io/blockjoy/node-base:abc123 -ENV DEBIAN_FRONTEND=noninteractive +# Add build arguments for Cloudflare and Grafana push credentials +ARG CLOUDFLARE_API_KEY +ARG GRAFANA_LOKI_BASICAUTH +ARG GRAFANA_PROM_BASICAUTH -# Install build dependencies -RUN apt-get update && apt-get install -y \ - clang \ - libclang-dev \ - llvm-dev \ - && rm -rf /var/lib/apt/lists/* +ENV DEBIAN_FRONTEND=noninteractive +ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} +ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} +ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} -ENV RETH_VERSION=v1.1.2 +ENV RETH_VERSION=v0.1.0-alpha.11 ENV CARGO_NET_GIT_FETCH_WITH_CLI=true # Install Rust and required dependencies diff --git a/clients/reth/config-alloy.template b/clients/reth/config-alloy.template index 56d348bf..78c5a80c 100644 --- a/clients/reth/config-alloy.template +++ b/clients/reth/config-alloy.template @@ -105,7 +105,7 @@ prometheus.remote_write "default" { basic_auth { username = "1234567" - password = ${GRAFANA_PROM_BASICAUTH} + password = "${GRAFANA_PROM_BASICAUTH}" } } } diff --git a/node-base/Dockerfile b/node-base/Dockerfile index 5d8e52a6..112bb9d0 100644 --- a/node-base/Dockerfile +++ b/node-base/Dockerfile @@ -1,14 +1,9 @@ FROM debian:bookworm-slim@sha256:a165446a88794db4fec31e35e9441433f9552ae048fb1ed26df352d2b537cb96 -# Add build arguments for Grafana credentials -ARG GRAFANA_LOKI_BASICAUTH -ARG GRAFANA_PROM_BASICAUTH ARG GRAFANA_LOKI_API_KEY ARG GRAFANA_PROM_API_KEY ENV DEBIAN_FRONTEND=noninteractive -ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} -ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} ENV GRAFANA_LOKI_API_KEY=${GRAFANA_LOKI_API_KEY} ENV GRAFANA_PROM_API_KEY=${GRAFANA_PROM_API_KEY} diff --git a/node-base/config/grafana-datasource.yml b/node-base/config/grafana-datasource.yml index 78605a4f..ae35edbe 100644 --- a/node-base/config/grafana-datasource.yml +++ b/node-base/config/grafana-datasource.yml @@ -8,7 +8,7 @@ datasources: basicAuth: true basicAuthUser: 1322871 secureJsonData: - basicAuthPassword: ${GRAFANA_PROM_API_KEY} + basicAuthPassword: "${GRAFANA_PROM_API_KEY}" - name: grafanacloud-logs type: loki url: https://logs-prod-012.grafana.net @@ -17,4 +17,4 @@ datasources: basicAuth: true basicAuthUser: 760471 secureJsonData: - basicAuthPassword: ${GRAFANA_LOKI_API_KEY} + basicAuthPassword: "${GRAFANA_LOKI_API_KEY}" From 21776204b29333e301581c1bbfbaea6504d7e347 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 25 Nov 2024 16:14:18 +0200 Subject: [PATCH 03/88] Clean up --- protocols.json | 1 - 1 file changed, 1 deletion(-) delete mode 100644 protocols.json diff --git a/protocols.json b/protocols.json deleted file mode 100644 index 04d895c8..00000000 --- a/protocols.json +++ /dev/null @@ -1 +0,0 @@ -// No longer used!!! All protocols are now tracked via the proto-tracker API. \ No newline at end of file From 91b32ddc061cfe4facabba51f6fb284ac9049582 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 25 Nov 2024 16:28:53 +0200 Subject: [PATCH 04/88] Fixed build workflow --- .github/workflows/docker-build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build-test.yml b/.github/workflows/docker-build-test.yml index b411166c..f4fcf5b8 100644 --- a/.github/workflows/docker-build-test.yml +++ b/.github/workflows/docker-build-test.yml @@ -46,7 +46,7 @@ jobs: if [[ "${{ matrix.image_path }}" == clients/* ]]; then IMAGE_NAME="blockjoy-$(basename ${{ matrix.image_path }})" if [[ -f "${{ matrix.image_path }}/Dockerfile" ]]; then - CLIENT_VERSION=$(grep -E "ENV.*VERSION[[:space:]]*[=][[:space:]]*v?[0-9]+\.[0-9]+\.[0-9]+" "${{ matrix.image_path }}/Dockerfile" | grep -oE "v?[0-9]+\.[0-9]+\.[0-9]+") + CLIENT_VERSION=$(grep -E "ENV.*_VERSION[[:space:]]*v?[0-9]+\.[0-9]+\.[0-9]+" "${{ matrix.image_path }}/Dockerfile" | grep -oE "v?[0-9]+\.[0-9]+\.[0-9]+") if [[ ! -z "$CLIENT_VERSION" ]]; then CLIENT_VERSION=${CLIENT_VERSION#v} IMAGE_TAG="${CLIENT_VERSION}-$(git rev-parse --short HEAD)" From ca6d53e7499dec05698a4795a0228dec5477f1ae Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 25 Nov 2024 16:34:02 +0200 Subject: [PATCH 05/88] Fixed Dockerfiles client versioning, build workflow regex --- .github/workflows/docker-build-test.yml | 2 +- clients/erigon/Dockerfile | 2 +- clients/reth/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-build-test.yml b/.github/workflows/docker-build-test.yml index f4fcf5b8..3217f06a 100644 --- a/.github/workflows/docker-build-test.yml +++ b/.github/workflows/docker-build-test.yml @@ -46,7 +46,7 @@ jobs: if [[ "${{ matrix.image_path }}" == clients/* ]]; then IMAGE_NAME="blockjoy-$(basename ${{ matrix.image_path }})" if [[ -f "${{ matrix.image_path }}/Dockerfile" ]]; then - CLIENT_VERSION=$(grep -E "ENV.*_VERSION[[:space:]]*v?[0-9]+\.[0-9]+\.[0-9]+" "${{ matrix.image_path }}/Dockerfile" | grep -oE "v?[0-9]+\.[0-9]+\.[0-9]+") + CLIENT_VERSION=$(grep -E "ENV.*_VERSION=[[:space:]]*v?[0-9]+\.[0-9]+\.[0-9]+[-.a-zA-Z0-9]*" "${{ matrix.image_path }}/Dockerfile" | grep -oE "v?[0-9]+\.[0-9]+\.[0-9]+[-.a-zA-Z0-9]*") if [[ ! -z "$CLIENT_VERSION" ]]; then CLIENT_VERSION=${CLIENT_VERSION#v} IMAGE_TAG="${CLIENT_VERSION}-$(git rev-parse --short HEAD)" diff --git a/clients/erigon/Dockerfile b/clients/erigon/Dockerfile index 55aed017..02685836 100644 --- a/clients/erigon/Dockerfile +++ b/clients/erigon/Dockerfile @@ -10,7 +10,7 @@ ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} -ENV ERIGON_VERSION v2.60.9 +ENV ERIGON_VERSION=v2.60.9 ENV PATH="/usr/local/go/bin:${PATH}" ENV GOROOT="/usr/local/go" diff --git a/clients/reth/Dockerfile b/clients/reth/Dockerfile index e5f92c0e..34f16220 100644 --- a/clients/reth/Dockerfile +++ b/clients/reth/Dockerfile @@ -10,7 +10,7 @@ ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} -ENV RETH_VERSION=v0.1.0-alpha.11 +ENV RETH_VERSION=v1.1.2 ENV CARGO_NET_GIT_FETCH_WITH_CLI=true # Install Rust and required dependencies From 49b208d7c60559c1f8a392b275ae69088c326ac8 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 25 Nov 2024 16:39:56 +0200 Subject: [PATCH 06/88] Fixed workflow auth --- .github/workflows/docker-build-test.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/docker-build-test.yml b/.github/workflows/docker-build-test.yml index 3217f06a..8f810b2b 100644 --- a/.github/workflows/docker-build-test.yml +++ b/.github/workflows/docker-build-test.yml @@ -40,6 +40,13 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Generate version id: version run: | From d085a04a6b88bf5b6b54a8b3bc09ff4b8785bab3 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 25 Nov 2024 17:09:24 +0200 Subject: [PATCH 07/88] Improved gha workflow --- ...docker-build-test.yml => docker-build.yml} | 50 +++++++--- .github/workflows/docker-publish.yml | 95 ------------------- 2 files changed, 39 insertions(+), 106 deletions(-) rename .github/workflows/{docker-build-test.yml => docker-build.yml} (58%) delete mode 100644 .github/workflows/docker-publish.yml diff --git a/.github/workflows/docker-build-test.yml b/.github/workflows/docker-build.yml similarity index 58% rename from .github/workflows/docker-build-test.yml rename to .github/workflows/docker-build.yml index 8f810b2b..19ff7a2d 100644 --- a/.github/workflows/docker-build-test.yml +++ b/.github/workflows/docker-build.yml @@ -1,4 +1,4 @@ -name: Docker Build Test +name: Docker Build on: pull_request: @@ -6,7 +6,6 @@ on: paths: - '**/Dockerfile' -# Configure GITHUB_TOKEN permissions permissions: contents: read packages: read @@ -28,8 +27,33 @@ jobs: - name: Generate build matrix id: set-matrix run: | - DIRS=$(find . -name Dockerfile -exec dirname {} \; | sed 's|^\./||' | jq -R -s -c 'split("\n")[:-1]') - echo "matrix=${DIRS}" >> $GITHUB_OUTPUT + # Order: node-base first, then clients, then protocols + MATRIX_DIRS=() + + # Add node-base if it exists + if [ -f "node-base/Dockerfile" ]; then + MATRIX_DIRS+=("node-base") + fi + + # Add client directories + CLIENT_DIRS=$(find ./clients -name Dockerfile -exec dirname {} \; | sed 's|^\./||' || true) + if [ ! -z "$CLIENT_DIRS" ]; then + while IFS= read -r dir; do + MATRIX_DIRS+=("$dir") + done <<< "$CLIENT_DIRS" + fi + + # Add protocol directories + PROTOCOL_DIRS=$(find . -name Dockerfile -not -path "./node-base/*" -not -path "./clients/*" -exec dirname {} \; | sed 's|^\./||' || true) + if [ ! -z "$PROTOCOL_DIRS" ]; then + while IFS= read -r dir; do + MATRIX_DIRS+=("$dir") + done <<< "$PROTOCOL_DIRS" + fi + + # Convert to JSON array + printf '%s\n' "${MATRIX_DIRS[@]}" | jq -R . | jq -s . > matrix.json + echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT build-test: needs: test-build @@ -50,9 +74,14 @@ jobs: - name: Generate version id: version run: | - if [[ "${{ matrix.image_path }}" == clients/* ]]; then - IMAGE_NAME="blockjoy-$(basename ${{ matrix.image_path }})" + if [[ "${{ matrix.image_path }}" == "node-base" ]]; then + IMAGE_NAME="node-base" + IMAGE_TAG="$(git rev-parse --short HEAD)" + elif [[ "${{ matrix.image_path }}" == clients/* ]]; then + CLIENT_NAME=$(basename ${{ matrix.image_path }}) + IMAGE_NAME="blockjoy-${CLIENT_NAME}" if [[ -f "${{ matrix.image_path }}/Dockerfile" ]]; then + # Extract client version from Dockerfile (e.g., ERIGON_VERSION, RETH_VERSION) CLIENT_VERSION=$(grep -E "ENV.*_VERSION=[[:space:]]*v?[0-9]+\.[0-9]+\.[0-9]+[-.a-zA-Z0-9]*" "${{ matrix.image_path }}/Dockerfile" | grep -oE "v?[0-9]+\.[0-9]+\.[0-9]+[-.a-zA-Z0-9]*") if [[ ! -z "$CLIENT_VERSION" ]]; then CLIENT_VERSION=${CLIENT_VERSION#v} @@ -73,16 +102,15 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Test build image + - name: Build and push images uses: docker/build-push-action@v5 with: context: ./${{ matrix.image_path }} - push: false - load: true + push: true build-args: | - ${{ startsWith(matrix.image_path, 'node-base') && format('GRAFANA_LOKI_API_KEY={0} + ${{ matrix.image_path == 'node-base' && format('GRAFANA_LOKI_API_KEY={0} GRAFANA_PROM_API_KEY={1}', secrets.GRAFANA_LOKI_API_KEY, secrets.GRAFANA_PROM_API_KEY) || '' }} - ${{ startsWith(matrix.image_path, 'clients') && format('GRAFANA_LOKI_BASICAUTH={0} + ${{ (startsWith(matrix.image_path, 'clients/') || !startsWith(matrix.image_path, 'node-base/')) && format('GRAFANA_LOKI_BASICAUTH={0} GRAFANA_PROM_BASICAUTH={1} CLOUDFLARE_API_KEY={2}', secrets.GRAFANA_LOKI_BASICAUTH, secrets.GRAFANA_PROM_BASICAUTH, secrets.CLOUDFLARE_API_KEY) || '' }} cache-from: type=gha diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml deleted file mode 100644 index 14222040..00000000 --- a/.github/workflows/docker-publish.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Docker Build and Publish - -on: - push: - branches: [ main ] - paths: - - '**/Dockerfile' - workflow_dispatch: - -# Configure GITHUB_TOKEN permissions -permissions: - contents: read - packages: write - pull-requests: write - id-token: write - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - build-and-publish: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - uses: actions/checkout@v4 - - - name: Generate build matrix - id: set-matrix - run: | - DIRS=$(find . -name Dockerfile -exec dirname {} \; | sed 's|^\./||' | jq -R -s -c 'split("\n")[:-1]') - echo "matrix=${DIRS}" >> $GITHUB_OUTPUT - - publish: - needs: build-and-publish - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - pull-requests: write - id-token: write - strategy: - matrix: - image_path: ${{ fromJson(needs.build-and-publish.outputs.matrix) }} - steps: - - uses: actions/checkout@v4 - - - name: Generate version - id: version - run: | - if [[ "${{ matrix.image_path }}" == clients/* ]]; then - IMAGE_NAME="blockjoy-$(basename ${{ matrix.image_path }})" - if [[ -f "${{ matrix.image_path }}/Dockerfile" ]]; then - CLIENT_VERSION=$(grep -E "ENV.*VERSION[[:space:]]*[=][[:space:]]*v?[0-9]+\.[0-9]+\.[0-9]+" "${{ matrix.image_path }}/Dockerfile" | grep -oE "v?[0-9]+\.[0-9]+\.[0-9]+") - if [[ ! -z "$CLIENT_VERSION" ]]; then - CLIENT_VERSION=${CLIENT_VERSION#v} - IMAGE_TAG="${CLIENT_VERSION}-$(git rev-parse --short HEAD)" - else - IMAGE_TAG="$(git rev-parse --short HEAD)" - fi - else - IMAGE_TAG="$(git rev-parse --short HEAD)" - fi - else - IMAGE_NAME=$(echo ${{ matrix.image_path }} | tr '/' '-') - IMAGE_TAG="$(git rev-parse --short HEAD)" - fi - echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT - echo "image_tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push image - uses: docker/build-push-action@v5 - with: - context: ./${{ matrix.image_path }} - push: true - build-args: | - ${{ startsWith(matrix.image_path, 'node-base') && format('GRAFANA_LOKI_API_KEY={0} - GRAFANA_PROM_API_KEY={1}', secrets.GRAFANA_LOKI_API_KEY, secrets.GRAFANA_PROM_API_KEY) || '' }} - ${{ startsWith(matrix.image_path, 'clients') && format('GRAFANA_LOKI_BASICAUTH={0} - GRAFANA_PROM_BASICAUTH={1} - CLOUDFLARE_API_KEY={2}', secrets.GRAFANA_LOKI_BASICAUTH, secrets.GRAFANA_PROM_BASICAUTH, secrets.CLOUDFLARE_API_KEY) || '' }} - cache-from: type=gha - cache-to: type=gha,mode=max - tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} From 94f1c4d43303f6490c17224ddf8007cc65397d5d Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 25 Nov 2024 17:12:47 +0200 Subject: [PATCH 08/88] Only build changed images --- .github/workflows/docker-build.yml | 74 +++++++++++++++++++----------- 1 file changed, 48 insertions(+), 26 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 19ff7a2d..feb70c2f 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -17,50 +17,72 @@ concurrency: cancel-in-progress: true jobs: - test-build: + detect-changes: runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Get changed Dockerfiles + id: changed-files + uses: tj-actions/changed-files@v40 + with: + files: | + **/Dockerfile + - name: Generate build matrix id: set-matrix run: | + # Get list of changed Dockerfiles + CHANGED_FILES="${{ steps.changed-files.outputs.all_changed_files }}" + # Order: node-base first, then clients, then protocols MATRIX_DIRS=() - # Add node-base if it exists - if [ -f "node-base/Dockerfile" ]; then - MATRIX_DIRS+=("node-base") - fi - - # Add client directories - CLIENT_DIRS=$(find ./clients -name Dockerfile -exec dirname {} \; | sed 's|^\./||' || true) - if [ ! -z "$CLIENT_DIRS" ]; then - while IFS= read -r dir; do - MATRIX_DIRS+=("$dir") - done <<< "$CLIENT_DIRS" - fi - - # Add protocol directories - PROTOCOL_DIRS=$(find . -name Dockerfile -not -path "./node-base/*" -not -path "./clients/*" -exec dirname {} \; | sed 's|^\./||' || true) - if [ ! -z "$PROTOCOL_DIRS" ]; then - while IFS= read -r dir; do - MATRIX_DIRS+=("$dir") - done <<< "$PROTOCOL_DIRS" - fi + # Process each changed Dockerfile + while IFS= read -r file; do + if [ -z "$file" ]; then + continue + fi + + # Get directory containing Dockerfile + DIR=$(dirname "$file") + + # Store based on priority + if [[ "$DIR" == "node-base" ]]; then + # node-base goes first if changed + MATRIX_DIRS=("node-base" "${MATRIX_DIRS[@]}") + elif [[ "$DIR" == clients/* ]]; then + # Add client dirs after node-base + if [[ " ${MATRIX_DIRS[@]} " =~ " node-base " ]]; then + MATRIX_DIRS+=("$DIR") + else + MATRIX_DIRS=("$DIR" "${MATRIX_DIRS[@]}") + fi + else + # Add protocol dirs last + MATRIX_DIRS+=("$DIR") + fi + done <<< "$CHANGED_FILES" # Convert to JSON array - printf '%s\n' "${MATRIX_DIRS[@]}" | jq -R . | jq -s . > matrix.json - echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT + if [ ${#MATRIX_DIRS[@]} -eq 0 ]; then + echo "matrix=[]" >> $GITHUB_OUTPUT + else + printf '%s\n' "${MATRIX_DIRS[@]}" | jq -R . | jq -s . > matrix.json + echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT + fi - build-test: - needs: test-build + build: + needs: detect-changes + if: ${{ needs.detect-changes.outputs.matrix != '[]' }} runs-on: ubuntu-latest strategy: matrix: - image_path: ${{ fromJson(needs.test-build.outputs.matrix) }} + image_path: ${{ fromJson(needs.detect-changes.outputs.matrix) }} steps: - uses: actions/checkout@v4 From 851623fa70b509db7e62b89c99208939b9880527 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 25 Nov 2024 17:14:50 +0200 Subject: [PATCH 09/88] Fixed changes list --- .github/workflows/docker-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index feb70c2f..065c1d0e 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -36,8 +36,8 @@ jobs: - name: Generate build matrix id: set-matrix run: | - # Get list of changed Dockerfiles - CHANGED_FILES="${{ steps.changed-files.outputs.all_changed_files }}" + # Get list of changed Dockerfiles and convert to newline-separated + CHANGED_FILES=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n') # Order: node-base first, then clients, then protocols MATRIX_DIRS=() From 454b98a0580f9e05a8e5251dd4ccf6b6c27a7dd0 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 25 Nov 2024 17:17:06 +0200 Subject: [PATCH 10/88] Fixed json formatting --- .github/workflows/docker-build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 065c1d0e..53e8b917 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -72,8 +72,9 @@ jobs: if [ ${#MATRIX_DIRS[@]} -eq 0 ]; then echo "matrix=[]" >> $GITHUB_OUTPUT else - printf '%s\n' "${MATRIX_DIRS[@]}" | jq -R . | jq -s . > matrix.json - echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT + # Create proper JSON array string + JSON_ARRAY=$(printf '%s\n' "${MATRIX_DIRS[@]}" | jq -R . | jq -sc .) + echo "matrix=$JSON_ARRAY" >> $GITHUB_OUTPUT fi build: From 9a0d5560fce3eea11febf324b8c4ad4c238f63fc Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 25 Nov 2024 17:23:57 +0200 Subject: [PATCH 11/88] Fixed bootstrap --- .github/workflows/docker-build.yml | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 53e8b917..5d4516d1 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -7,9 +7,9 @@ on: - '**/Dockerfile' permissions: - contents: read - packages: read - pull-requests: read + contents: write + packages: write + pull-requests: write id-token: write concurrency: @@ -139,3 +139,27 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} + + - name: Update dependent Dockerfiles + if: success() && (matrix.image_path == 'node-base' || startsWith(matrix.image_path, 'clients/')) + run: | + # Configure git + git config --global user.name "GitHub Actions" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + # Update dependent Dockerfiles based on what was just built + if [[ "${{ matrix.image_path }}" == "node-base" ]]; then + # Update all client Dockerfiles with new node-base tag + find ./clients -name Dockerfile -type f -exec sed -i "s|FROM ghcr.io/blockjoy/node-base:[^ ]*|FROM ghcr.io/blockjoy/node-base:${{ steps.version.outputs.image_tag }}|g" {} \; + elif [[ "${{ matrix.image_path }}" == clients/* ]]; then + # Get client name and update corresponding protocol Dockerfiles + CLIENT_NAME=$(basename ${{ matrix.image_path }}) + find . -name Dockerfile -not -path "./node-base/*" -not -path "./clients/*" -type f -exec sed -i "s|FROM ghcr.io/blockjoy/blockjoy-${CLIENT_NAME}:[^ ]*|FROM ghcr.io/blockjoy/blockjoy-${CLIENT_NAME}:${{ steps.version.outputs.image_tag }}|g" {} \; + fi + + # Check if there are changes to commit + if [[ -n "$(git status --porcelain)" ]]; then + git add -A + git commit -m "chore: update Dockerfile dependencies for ${{ matrix.image_path }}" + git push origin HEAD:${{ github.head_ref }} + fi From dd418d1f9313517d5ba840ef84d423feb8160bfe Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 25 Nov 2024 17:28:47 +0200 Subject: [PATCH 12/88] Added renovate step --- .github/workflows/docker-build.yml | 36 ++++++++++++++++-------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 5d4516d1..1cb28952 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -7,9 +7,9 @@ on: - '**/Dockerfile' permissions: - contents: write + contents: read packages: write - pull-requests: write + pull-requests: read id-token: write concurrency: @@ -140,26 +140,28 @@ jobs: cache-to: type=gha,mode=max tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} - - name: Update dependent Dockerfiles + - name: Trigger Renovate update if: success() && (matrix.image_path == 'node-base' || startsWith(matrix.image_path, 'clients/')) + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - # Configure git - git config --global user.name "GitHub Actions" - git config --global user.email "github-actions[bot]@users.noreply.github.com" + # Wait a bit for the image to be available + sleep 10 - # Update dependent Dockerfiles based on what was just built + # Determine which paths to update based on what was built if [[ "${{ matrix.image_path }}" == "node-base" ]]; then - # Update all client Dockerfiles with new node-base tag - find ./clients -name Dockerfile -type f -exec sed -i "s|FROM ghcr.io/blockjoy/node-base:[^ ]*|FROM ghcr.io/blockjoy/node-base:${{ steps.version.outputs.image_tag }}|g" {} \; + PATHS="clients/**/Dockerfile" elif [[ "${{ matrix.image_path }}" == clients/* ]]; then - # Get client name and update corresponding protocol Dockerfiles CLIENT_NAME=$(basename ${{ matrix.image_path }}) - find . -name Dockerfile -not -path "./node-base/*" -not -path "./clients/*" -type f -exec sed -i "s|FROM ghcr.io/blockjoy/blockjoy-${CLIENT_NAME}:[^ ]*|FROM ghcr.io/blockjoy/blockjoy-${CLIENT_NAME}:${{ steps.version.outputs.image_tag }}|g" {} \; + PATHS="ethereum/**/Dockerfile" fi - # Check if there are changes to commit - if [[ -n "$(git status --porcelain)" ]]; then - git add -A - git commit -m "chore: update Dockerfile dependencies for ${{ matrix.image_path }}" - git push origin HEAD:${{ github.head_ref }} - fi + # Create repository issue to trigger Renovate + gh issue create \ + --title "build: update dependencies for ${{ matrix.image_path }}" \ + --body "renovate:enableVulnerabilityAlerts + + Update dependencies for newly built image ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} + + Paths to update: ${PATHS}" \ + --label "dependencies" From ed9135d16121880a5dd2ad92562f2b1a5aa10ac5 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 25 Nov 2024 17:36:46 +0200 Subject: [PATCH 13/88] Improved workflow --- .github/workflows/docker-build.yml | 173 ++++++++++++++++------------- 1 file changed, 93 insertions(+), 80 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 1cb28952..82013c91 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -20,7 +20,9 @@ jobs: detect-changes: runs-on: ubuntu-latest outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + base_matrix: ${{ steps.set-matrix.outputs.base_matrix }} + clients_matrix: ${{ steps.set-matrix.outputs.clients_matrix }} + protocols_matrix: ${{ steps.set-matrix.outputs.protocols_matrix }} steps: - uses: actions/checkout@v4 with: @@ -33,14 +35,16 @@ jobs: files: | **/Dockerfile - - name: Generate build matrix + - name: Generate build matrices id: set-matrix run: | # Get list of changed Dockerfiles and convert to newline-separated CHANGED_FILES=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n') - # Order: node-base first, then clients, then protocols - MATRIX_DIRS=() + # Initialize empty arrays for each group + BASE_MATRIX='{"include":[]}' + CLIENTS_MATRIX='{"include":[]}' + PROTOCOLS_MATRIX='{"include":[]}' # Process each changed Dockerfile while IFS= read -r file; do @@ -51,117 +55,126 @@ jobs: # Get directory containing Dockerfile DIR=$(dirname "$file") - # Store based on priority + # Add to appropriate matrix if [[ "$DIR" == "node-base" ]]; then - # node-base goes first if changed - MATRIX_DIRS=("node-base" "${MATRIX_DIRS[@]}") + BASE_MATRIX=$(echo $BASE_MATRIX | jq --arg path "$DIR" '.include += [{"image_path": $path}]') elif [[ "$DIR" == clients/* ]]; then - # Add client dirs after node-base - if [[ " ${MATRIX_DIRS[@]} " =~ " node-base " ]]; then - MATRIX_DIRS+=("$DIR") - else - MATRIX_DIRS=("$DIR" "${MATRIX_DIRS[@]}") - fi + CLIENTS_MATRIX=$(echo $CLIENTS_MATRIX | jq --arg path "$DIR" '.include += [{"image_path": $path}]') else - # Add protocol dirs last - MATRIX_DIRS+=("$DIR") + PROTOCOLS_MATRIX=$(echo $PROTOCOLS_MATRIX | jq --arg path "$DIR" '.include += [{"image_path": $path}]') fi done <<< "$CHANGED_FILES" - # Convert to JSON array - if [ ${#MATRIX_DIRS[@]} -eq 0 ]; then - echo "matrix=[]" >> $GITHUB_OUTPUT - else - # Create proper JSON array string - JSON_ARRAY=$(printf '%s\n' "${MATRIX_DIRS[@]}" | jq -R . | jq -sc .) - echo "matrix=$JSON_ARRAY" >> $GITHUB_OUTPUT - fi + echo "base_matrix=$BASE_MATRIX" >> $GITHUB_OUTPUT + echo "clients_matrix=$CLIENTS_MATRIX" >> $GITHUB_OUTPUT + echo "protocols_matrix=$PROTOCOLS_MATRIX" >> $GITHUB_OUTPUT - build: + build-base: needs: detect-changes - if: ${{ needs.detect-changes.outputs.matrix != '[]' }} + if: ${{ fromJson(needs.detect-changes.outputs.base_matrix).include[0] }} runs-on: ubuntu-latest strategy: - matrix: - image_path: ${{ fromJson(needs.detect-changes.outputs.matrix) }} + matrix: ${{ fromJson(needs.detect-changes.outputs.base_matrix) }} + steps: + - uses: actions/checkout@v4 + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push base image + uses: docker/build-push-action@v5 + with: + context: ./${{ matrix.image_path }} + push: true + build-args: | + GRAFANA_LOKI_API_KEY=${{ secrets.GRAFANA_LOKI_API_KEY }} + GRAFANA_PROM_API_KEY=${{ secrets.GRAFANA_PROM_API_KEY }} + cache-from: type=gha + cache-to: type=gha,mode=max + tags: ghcr.io/blockjoy/node-base:latest + + build-clients: + needs: [detect-changes, build-base] + if: | + always() && + needs.build-base.result != 'failure' && + fromJson(needs.detect-changes.outputs.clients_matrix).include[0] + runs-on: ubuntu-latest + strategy: + matrix: ${{ fromJson(needs.detect-changes.outputs.clients_matrix) }} steps: - uses: actions/checkout@v4 - - name: Log in to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Generate version id: version run: | - if [[ "${{ matrix.image_path }}" == "node-base" ]]; then - IMAGE_NAME="node-base" - IMAGE_TAG="$(git rev-parse --short HEAD)" - elif [[ "${{ matrix.image_path }}" == clients/* ]]; then - CLIENT_NAME=$(basename ${{ matrix.image_path }}) - IMAGE_NAME="blockjoy-${CLIENT_NAME}" - if [[ -f "${{ matrix.image_path }}/Dockerfile" ]]; then - # Extract client version from Dockerfile (e.g., ERIGON_VERSION, RETH_VERSION) - CLIENT_VERSION=$(grep -E "ENV.*_VERSION=[[:space:]]*v?[0-9]+\.[0-9]+\.[0-9]+[-.a-zA-Z0-9]*" "${{ matrix.image_path }}/Dockerfile" | grep -oE "v?[0-9]+\.[0-9]+\.[0-9]+[-.a-zA-Z0-9]*") - if [[ ! -z "$CLIENT_VERSION" ]]; then - CLIENT_VERSION=${CLIENT_VERSION#v} - IMAGE_TAG="${CLIENT_VERSION}-$(git rev-parse --short HEAD)" - else - IMAGE_TAG="$(git rev-parse --short HEAD)" - fi + CLIENT_NAME=$(basename ${{ matrix.image_path }}) + IMAGE_NAME="blockjoy-${CLIENT_NAME}" + if [[ -f "${{ matrix.image_path }}/Dockerfile" ]]; then + CLIENT_VERSION=$(grep -E "ENV.*_VERSION=[[:space:]]*v?[0-9]+\.[0-9]+\.[0-9]+[-.a-zA-Z0-9]*" "${{ matrix.image_path }}/Dockerfile" | grep -oE "v?[0-9]+\.[0-9]+\.[0-9]+[-.a-zA-Z0-9]*") + if [[ ! -z "$CLIENT_VERSION" ]]; then + CLIENT_VERSION=${CLIENT_VERSION#v} + IMAGE_TAG="${CLIENT_VERSION}-latest" else - IMAGE_TAG="$(git rev-parse --short HEAD)" + IMAGE_TAG="latest" fi else - IMAGE_NAME=$(echo ${{ matrix.image_path }} | tr '/' '-') - IMAGE_TAG="$(git rev-parse --short HEAD)" + IMAGE_TAG="latest" fi echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT echo "image_tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - - name: Build and push images + - name: Build and push client images uses: docker/build-push-action@v5 with: context: ./${{ matrix.image_path }} push: true build-args: | - ${{ matrix.image_path == 'node-base' && format('GRAFANA_LOKI_API_KEY={0} - GRAFANA_PROM_API_KEY={1}', secrets.GRAFANA_LOKI_API_KEY, secrets.GRAFANA_PROM_API_KEY) || '' }} - ${{ (startsWith(matrix.image_path, 'clients/') || !startsWith(matrix.image_path, 'node-base/')) && format('GRAFANA_LOKI_BASICAUTH={0} - GRAFANA_PROM_BASICAUTH={1} - CLOUDFLARE_API_KEY={2}', secrets.GRAFANA_LOKI_BASICAUTH, secrets.GRAFANA_PROM_BASICAUTH, secrets.CLOUDFLARE_API_KEY) || '' }} + GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }} + GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }} + CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }} cache-from: type=gha cache-to: type=gha,mode=max tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} - - name: Trigger Renovate update - if: success() && (matrix.image_path == 'node-base' || startsWith(matrix.image_path, 'clients/')) - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Wait a bit for the image to be available - sleep 10 - - # Determine which paths to update based on what was built - if [[ "${{ matrix.image_path }}" == "node-base" ]]; then - PATHS="clients/**/Dockerfile" - elif [[ "${{ matrix.image_path }}" == clients/* ]]; then - CLIENT_NAME=$(basename ${{ matrix.image_path }}) - PATHS="ethereum/**/Dockerfile" - fi - - # Create repository issue to trigger Renovate - gh issue create \ - --title "build: update dependencies for ${{ matrix.image_path }}" \ - --body "renovate:enableVulnerabilityAlerts - - Update dependencies for newly built image ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} - - Paths to update: ${PATHS}" \ - --label "dependencies" + build-protocols: + needs: [detect-changes, build-clients] + if: | + always() && + needs.build-clients.result != 'failure' && + fromJson(needs.detect-changes.outputs.protocols_matrix).include[0] + runs-on: ubuntu-latest + strategy: + matrix: ${{ fromJson(needs.detect-changes.outputs.protocols_matrix) }} + steps: + - uses: actions/checkout@v4 + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push protocol images + uses: docker/build-push-action@v5 + with: + context: ./${{ matrix.image_path }} + push: true + build-args: | + GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }} + GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }} + CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }} + cache-from: type=gha + cache-to: type=gha,mode=max + tags: ghcr.io/blockjoy/${{ matrix.image_path | replace('/', '-') }}:latest From 16d3d55a4ec7f5b894b9067779acbedf4d605a8e Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 25 Nov 2024 17:44:40 +0200 Subject: [PATCH 14/88] Improved versioning --- .github/workflows/docker-build.yml | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 82013c91..3ccf2530 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -83,6 +83,12 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Generate version + id: version + run: | + SHA=$(git rev-parse --short HEAD) + echo "image_name=node-base" >> $GITHUB_OUTPUT + echo "image_tag=${SHA}" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push base image @@ -95,7 +101,7 @@ jobs: GRAFANA_PROM_API_KEY=${{ secrets.GRAFANA_PROM_API_KEY }} cache-from: type=gha cache-to: type=gha,mode=max - tags: ghcr.io/blockjoy/node-base:latest + tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} build-clients: needs: [detect-changes, build-base] @@ -117,18 +123,20 @@ jobs: - name: Generate version id: version run: | + SHA=$(git rev-parse --short HEAD) CLIENT_NAME=$(basename ${{ matrix.image_path }}) IMAGE_NAME="blockjoy-${CLIENT_NAME}" + if [[ -f "${{ matrix.image_path }}/Dockerfile" ]]; then CLIENT_VERSION=$(grep -E "ENV.*_VERSION=[[:space:]]*v?[0-9]+\.[0-9]+\.[0-9]+[-.a-zA-Z0-9]*" "${{ matrix.image_path }}/Dockerfile" | grep -oE "v?[0-9]+\.[0-9]+\.[0-9]+[-.a-zA-Z0-9]*") if [[ ! -z "$CLIENT_VERSION" ]]; then CLIENT_VERSION=${CLIENT_VERSION#v} - IMAGE_TAG="${CLIENT_VERSION}-latest" + IMAGE_TAG="${CLIENT_VERSION}-${SHA}" else - IMAGE_TAG="latest" + IMAGE_TAG="${SHA}" fi else - IMAGE_TAG="latest" + IMAGE_TAG="${SHA}" fi echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT echo "image_tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT @@ -164,6 +172,13 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Generate version + id: version + run: | + SHA=$(git rev-parse --short HEAD) + IMAGE_NAME=$(basename $(dirname ${{ matrix.image_path }}))"-"$(basename ${{ matrix.image_path }}) + echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT + echo "image_tag=${SHA}" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push protocol images @@ -177,4 +192,4 @@ jobs: CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }} cache-from: type=gha cache-to: type=gha,mode=max - tags: ghcr.io/blockjoy/${{ matrix.image_path | replace('/', '-') }}:latest + tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} From a87c7770d293415e0bcb2c2c057293ac989e9d49 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 25 Nov 2024 17:47:56 +0200 Subject: [PATCH 15/88] JSON fixes --- .github/workflows/docker-build.yml | 35 +++++------------------------- 1 file changed, 6 insertions(+), 29 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 3ccf2530..a6d4ff45 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -38,36 +38,13 @@ jobs: - name: Generate build matrices id: set-matrix run: | - # Get list of changed Dockerfiles and convert to newline-separated - CHANGED_FILES=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n') + # Get list of changed Dockerfiles and convert to JSON array + CHANGED_FILES=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | jq -R -s 'split("\n")[:-1]') - # Initialize empty arrays for each group - BASE_MATRIX='{"include":[]}' - CLIENTS_MATRIX='{"include":[]}' - PROTOCOLS_MATRIX='{"include":[]}' - - # Process each changed Dockerfile - while IFS= read -r file; do - if [ -z "$file" ]; then - continue - fi - - # Get directory containing Dockerfile - DIR=$(dirname "$file") - - # Add to appropriate matrix - if [[ "$DIR" == "node-base" ]]; then - BASE_MATRIX=$(echo $BASE_MATRIX | jq --arg path "$DIR" '.include += [{"image_path": $path}]') - elif [[ "$DIR" == clients/* ]]; then - CLIENTS_MATRIX=$(echo $CLIENTS_MATRIX | jq --arg path "$DIR" '.include += [{"image_path": $path}]') - else - PROTOCOLS_MATRIX=$(echo $PROTOCOLS_MATRIX | jq --arg path "$DIR" '.include += [{"image_path": $path}]') - fi - done <<< "$CHANGED_FILES" - - echo "base_matrix=$BASE_MATRIX" >> $GITHUB_OUTPUT - echo "clients_matrix=$CLIENTS_MATRIX" >> $GITHUB_OUTPUT - echo "protocols_matrix=$PROTOCOLS_MATRIX" >> $GITHUB_OUTPUT + # Create matrices using jq filters + echo "base_matrix=$(echo $CHANGED_FILES | jq '{"include": map(select(startswith("node-base")) | {image_path: .})')" >> $GITHUB_OUTPUT + echo "clients_matrix=$(echo $CHANGED_FILES | jq '{"include": map(select(startswith("clients/")) | {image_path: split("/")[0:2]|join("/")})')" >> $GITHUB_OUTPUT + echo "protocols_matrix=$(echo $CHANGED_FILES | jq '{"include": map(select(startswith("ethereum/")) | {image_path: split("/")[0:2]|join("/")})')" >> $GITHUB_OUTPUT build-base: needs: detect-changes From b772579e2e1d85dde02d7e847fa910a1d111a7df Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 25 Nov 2024 17:49:40 +0200 Subject: [PATCH 16/88] JSON fixes --- .github/workflows/docker-build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index a6d4ff45..38302d23 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -42,9 +42,9 @@ jobs: CHANGED_FILES=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | jq -R -s 'split("\n")[:-1]') # Create matrices using jq filters - echo "base_matrix=$(echo $CHANGED_FILES | jq '{"include": map(select(startswith("node-base")) | {image_path: .})')" >> $GITHUB_OUTPUT - echo "clients_matrix=$(echo $CHANGED_FILES | jq '{"include": map(select(startswith("clients/")) | {image_path: split("/")[0:2]|join("/")})')" >> $GITHUB_OUTPUT - echo "protocols_matrix=$(echo $CHANGED_FILES | jq '{"include": map(select(startswith("ethereum/")) | {image_path: split("/")[0:2]|join("/")})')" >> $GITHUB_OUTPUT + echo "base_matrix=$(echo $CHANGED_FILES | jq '{include: map(select(startswith("node-base")) | {image_path: .}) }')" >> $GITHUB_OUTPUT + echo "clients_matrix=$(echo $CHANGED_FILES | jq '{include: map(select(startswith("clients/")) | {image_path: (split("/")[0:2]|join("/"))}) }')" >> $GITHUB_OUTPUT + echo "protocols_matrix=$(echo $CHANGED_FILES | jq '{include: map(select(startswith("ethereum/")) | {image_path: (split("/")[0:2]|join("/"))}) }')" >> $GITHUB_OUTPUT build-base: needs: detect-changes From effaf061d036c67c3956cfbf1a74c59bdbb769ad Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 25 Nov 2024 17:50:43 +0200 Subject: [PATCH 17/88] JSON fixes --- .github/workflows/docker-build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 38302d23..5b1dcb93 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -38,13 +38,13 @@ jobs: - name: Generate build matrices id: set-matrix run: | - # Get list of changed Dockerfiles and convert to JSON array - CHANGED_FILES=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | jq -R -s 'split("\n")[:-1]') + # Get list of changed Dockerfiles + CHANGED_FILES=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n') - # Create matrices using jq filters - echo "base_matrix=$(echo $CHANGED_FILES | jq '{include: map(select(startswith("node-base")) | {image_path: .}) }')" >> $GITHUB_OUTPUT - echo "clients_matrix=$(echo $CHANGED_FILES | jq '{include: map(select(startswith("clients/")) | {image_path: (split("/")[0:2]|join("/"))}) }')" >> $GITHUB_OUTPUT - echo "protocols_matrix=$(echo $CHANGED_FILES | jq '{include: map(select(startswith("ethereum/")) | {image_path: (split("/")[0:2]|join("/"))}) }')" >> $GITHUB_OUTPUT + # Create matrices using jq + echo "base_matrix=$(echo "$CHANGED_FILES" | grep "^node-base/" | sed 's|/Dockerfile$||' | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT + echo "clients_matrix=$(echo "$CHANGED_FILES" | grep "^clients/" | sed 's|/Dockerfile$||' | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT + echo "protocols_matrix=$(echo "$CHANGED_FILES" | grep "^ethereum/" | sed 's|/Dockerfile$||' | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT build-base: needs: detect-changes From d4f855e46f7a2f6fcbd8b1dcc5ace550e16342bf Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 25 Nov 2024 17:58:06 +0200 Subject: [PATCH 18/88] Fixed initial versioning --- clients/erigon/Dockerfile | 2 +- clients/reth/Dockerfile | 2 +- ethereum/ethereum-erigon/Dockerfile | 2 +- ethereum/ethereum-reth/Dockerfile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/clients/erigon/Dockerfile b/clients/erigon/Dockerfile index 02685836..28e7edbd 100644 --- a/clients/erigon/Dockerfile +++ b/clients/erigon/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/blockjoy/node-base:abc123 +FROM ghcr.io/blockjoy/node-base:latest # Add build arguments for Cloudflare and Grafana push credentials ARG CLOUDFLARE_API_KEY diff --git a/clients/reth/Dockerfile b/clients/reth/Dockerfile index 34f16220..aba81d20 100644 --- a/clients/reth/Dockerfile +++ b/clients/reth/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/blockjoy/node-base:abc123 +FROM ghcr.io/blockjoy/node-base:latest # Add build arguments for Cloudflare and Grafana push credentials ARG CLOUDFLARE_API_KEY diff --git a/ethereum/ethereum-erigon/Dockerfile b/ethereum/ethereum-erigon/Dockerfile index d3e858ad..d3218ae5 100644 --- a/ethereum/ethereum-erigon/Dockerfile +++ b/ethereum/ethereum-erigon/Dockerfile @@ -1,3 +1,3 @@ -FROM ghcr.io/blockjoy/blockjoy-erigon:def456 +FROM ghcr.io/blockjoy/blockjoy-erigon:latest COPY ./main.rhai /var/lib/babel/plugin/ diff --git a/ethereum/ethereum-reth/Dockerfile b/ethereum/ethereum-reth/Dockerfile index 98024573..3349e723 100644 --- a/ethereum/ethereum-reth/Dockerfile +++ b/ethereum/ethereum-reth/Dockerfile @@ -1,3 +1,3 @@ -FROM reth:v1.1.2 +FROM ghcr.io/blockjoy/blockjoy-reth:latest COPY ./main.rhai /var/lib/babel/plugin/ From 0d2ef84b465fb2f9b08674a7ab43243e26c9171d Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 25 Nov 2024 18:01:29 +0200 Subject: [PATCH 19/88] Manual versions --- clients/erigon/Dockerfile | 2 +- clients/reth/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/erigon/Dockerfile b/clients/erigon/Dockerfile index 28e7edbd..3134028c 100644 --- a/clients/erigon/Dockerfile +++ b/clients/erigon/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/blockjoy/node-base:latest +FROM ghcr.io/blockjoy/node-base:866a6bb@sha256:7923a8832df1119faf8d56cb501764ad9edb5bba6f54a2382edd0b8a2385bda3 # Add build arguments for Cloudflare and Grafana push credentials ARG CLOUDFLARE_API_KEY diff --git a/clients/reth/Dockerfile b/clients/reth/Dockerfile index aba81d20..76fbc1cf 100644 --- a/clients/reth/Dockerfile +++ b/clients/reth/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/blockjoy/node-base:latest +FROM ghcr.io/blockjoy/node-base:866a6bb@sha256:7923a8832df1119faf8d56cb501764ad9edb5bba6f54a2382edd0b8a2385bda3 # Add build arguments for Cloudflare and Grafana push credentials ARG CLOUDFLARE_API_KEY From ed53435cb6726904e45085ada4b1876aa86a31e7 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 25 Nov 2024 18:08:44 +0200 Subject: [PATCH 20/88] Reth dependencies --- clients/reth/Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/clients/reth/Dockerfile b/clients/reth/Dockerfile index 76fbc1cf..a72b14ac 100644 --- a/clients/reth/Dockerfile +++ b/clients/reth/Dockerfile @@ -13,6 +13,13 @@ ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} ENV RETH_VERSION=v1.1.2 ENV CARGO_NET_GIT_FETCH_WITH_CLI=true +# Install system dependencies including libclang +RUN apt-get update && \ + apt-get install -y \ + clang \ + libclang-dev \ + && rm -rf /var/lib/apt/lists/* + # Install Rust and required dependencies RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \ . "$HOME/.cargo/env" && \ From e4765d1858ab1fc134e93a91a95edc0dd3e40510 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 25 Nov 2024 18:31:17 +0200 Subject: [PATCH 21/88] Added versions manually --- ethereum/ethereum-erigon/Dockerfile | 2 +- ethereum/ethereum-reth/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ethereum/ethereum-erigon/Dockerfile b/ethereum/ethereum-erigon/Dockerfile index d3218ae5..be6a4291 100644 --- a/ethereum/ethereum-erigon/Dockerfile +++ b/ethereum/ethereum-erigon/Dockerfile @@ -1,3 +1,3 @@ -FROM ghcr.io/blockjoy/blockjoy-erigon:latest +FROM ghcr.io/blockjoy/blockjoy-erigon:2.60.9-e1fce25@sha256:d02854883030ac8555bfc4529668ab7e842a2225b0806cee97128ecbdfa22d3b COPY ./main.rhai /var/lib/babel/plugin/ diff --git a/ethereum/ethereum-reth/Dockerfile b/ethereum/ethereum-reth/Dockerfile index 3349e723..14067617 100644 --- a/ethereum/ethereum-reth/Dockerfile +++ b/ethereum/ethereum-reth/Dockerfile @@ -1,3 +1,3 @@ -FROM ghcr.io/blockjoy/blockjoy-reth:latest +FROM ghcr.io/blockjoy/blockjoy-reth:1.1.2-e1fce25@sha256:e0da7d51b0635352269d0ba70ecf918bbc46dc5539b1a1ef28bcc8ef0a9c2717 COPY ./main.rhai /var/lib/babel/plugin/ From d059b6077cf6102867a6862b4a46df4a7390ac0f Mon Sep 17 00:00:00 2001 From: mateipopa Date: Fri, 29 Nov 2024 16:51:18 +0200 Subject: [PATCH 22/88] Detect all changes --- .github/workflows/docker-build.yml | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 5b1dcb93..fe06b63d 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -3,8 +3,10 @@ name: Docker Build on: pull_request: branches: [ main ] - paths: - - '**/Dockerfile' + paths-ignore: + - '**.md' + - '.github/**' + - '!.github/workflows/docker-build.yml' permissions: contents: read @@ -28,23 +30,29 @@ jobs: with: fetch-depth: 0 - - name: Get changed Dockerfiles + - name: Get changed files id: changed-files uses: tj-actions/changed-files@v40 with: files: | - **/Dockerfile + ** - name: Generate build matrices id: set-matrix run: | - # Get list of changed Dockerfiles - CHANGED_FILES=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n') + # Get list of changed files and filter to unique directories containing Dockerfile + CHANGED_DIRS=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | while read -r file; do + dir=$(dirname "$file") + if [[ -f "$dir/Dockerfile" ]]; then + echo "$dir" + fi + done | sort -u) - # Create matrices using jq - echo "base_matrix=$(echo "$CHANGED_FILES" | grep "^node-base/" | sed 's|/Dockerfile$||' | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT - echo "clients_matrix=$(echo "$CHANGED_FILES" | grep "^clients/" | sed 's|/Dockerfile$||' | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT - echo "protocols_matrix=$(echo "$CHANGED_FILES" | grep "^ethereum/" | sed 's|/Dockerfile$||' | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT + # Create matrices using jq, filtering by directory structure + echo "base_matrix=$(echo "$CHANGED_DIRS" | grep "^node-base$" | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT + echo "clients_matrix=$(echo "$CHANGED_DIRS" | grep "^clients/" | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT + # Any directory that's not node-base or clients/ and contains a Dockerfile is a protocol + echo "protocols_matrix=$(echo "$CHANGED_DIRS" | grep -v "^node-base$" | grep -v "^clients/" | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT build-base: needs: detect-changes From 7b93220cd08697a59d1059f1bc5cf8e8e75d28d6 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Wed, 18 Dec 2024 15:20:14 +0200 Subject: [PATCH 23/88] Fixes --- ethereum/ethereum-reth/babel.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ethereum/ethereum-reth/babel.yaml b/ethereum/ethereum-reth/babel.yaml index e06aad11..b903e5f6 100644 --- a/ethereum/ethereum-reth/babel.yaml +++ b/ethereum/ethereum-reth/babel.yaml @@ -9,7 +9,7 @@ variants: # The same protocol can have different implementations. This field let identify what implementation(s) has been used. # Variant name may additional contain protocol network name e.g. "eth-mainnet". # This value will be available in a Rhai runtime via `node_env().variant_key`. - - key: reth_mainnet_archive + - key: reth-mainnet-archive min_cpu: 12 min_memory_mb: 32000 min_disk_gb: 3500 @@ -17,7 +17,7 @@ variants: - pointer: !store_id ethereum-reth-mainnet-archive-v1 - pointer: !store_id ethereum-reth-mainnet-archive-v1 new_archive_properties: [ "tracing-api-on-value" ] - - key: reth_sepolia_archive + - key: reth-sepolia-archive min_cpu: 6 min_memory_mb: 16000 min_disk_gb: 1000 @@ -25,7 +25,7 @@ variants: - pointer: !store_id ethereum-reth-sepolia-archive-v1 - pointer: !store_id ethereum-reth-sepolia-archive-v1 new_archive_properties: [ "tracing-api-on-value" ] - - key: reth_mainnet_full + - key: reth-mainnet-full min_cpu: 8 min_memory_mb: 16000 min_disk_gb: 1500 @@ -33,7 +33,7 @@ variants: - pointer: !store_id ethereum-reth-mainnet-full-v1 - pointer: !combination_disallowed new_archive_properties: [ "tracing-api-on-value" ] - - key: reth_sepolia_full + - key: reth-sepolia-full min_cpu: 4 min_memory_mb: 8000 min_disk_gb: 400 @@ -90,4 +90,6 @@ properties: add_disk_bytes: 10000000000 off: value: off-value - impact: null \ No newline at end of file + impact: null + + From 795bfd601b052d5ff48dcf445135f4f35fd68be0 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 19 Dec 2024 05:21:32 +0000 Subject: [PATCH 24/88] update to lates nib and add protocols.yaml --- clients/reth/Dockerfile | 3 +-- ethereum/ethereum-reth/Dockerfile | 2 +- ethereum/ethereum-reth/babel.yaml | 26 ++++++-------------------- ethereum/ethereum-reth/main.rhai | 16 ++++++++-------- 4 files changed, 16 insertions(+), 31 deletions(-) diff --git a/clients/reth/Dockerfile b/clients/reth/Dockerfile index a72b14ac..22776b8b 100644 --- a/clients/reth/Dockerfile +++ b/clients/reth/Dockerfile @@ -1,5 +1,4 @@ -FROM ghcr.io/blockjoy/node-base:866a6bb@sha256:7923a8832df1119faf8d56cb501764ad9edb5bba6f54a2382edd0b8a2385bda3 - +FROM node-base:latest # Add build arguments for Cloudflare and Grafana push credentials ARG CLOUDFLARE_API_KEY ARG GRAFANA_LOKI_BASICAUTH diff --git a/ethereum/ethereum-reth/Dockerfile b/ethereum/ethereum-reth/Dockerfile index 14067617..5c5f89b8 100644 --- a/ethereum/ethereum-reth/Dockerfile +++ b/ethereum/ethereum-reth/Dockerfile @@ -1,3 +1,3 @@ -FROM ghcr.io/blockjoy/blockjoy-reth:1.1.2-e1fce25@sha256:e0da7d51b0635352269d0ba70ecf918bbc46dc5539b1a1ef28bcc8ef0a9c2717 +FROM reth:latest COPY ./main.rhai /var/lib/babel/plugin/ diff --git a/ethereum/ethereum-reth/babel.yaml b/ethereum/ethereum-reth/babel.yaml index b903e5f6..37f12890 100644 --- a/ethereum/ethereum-reth/babel.yaml +++ b/ethereum/ethereum-reth/babel.yaml @@ -15,32 +15,24 @@ variants: min_disk_gb: 3500 archive_pointers: - pointer: !store_id ethereum-reth-mainnet-archive-v1 - - pointer: !store_id ethereum-reth-mainnet-archive-v1 - new_archive_properties: [ "tracing-api-on-value" ] - key: reth-sepolia-archive min_cpu: 6 min_memory_mb: 16000 min_disk_gb: 1000 archive_pointers: - pointer: !store_id ethereum-reth-sepolia-archive-v1 - - pointer: !store_id ethereum-reth-sepolia-archive-v1 - new_archive_properties: [ "tracing-api-on-value" ] - key: reth-mainnet-full min_cpu: 8 min_memory_mb: 16000 min_disk_gb: 1500 archive_pointers: - pointer: !store_id ethereum-reth-mainnet-full-v1 - - pointer: !combination_disallowed - new_archive_properties: [ "tracing-api-on-value" ] - key: reth-sepolia-full min_cpu: 4 min_memory_mb: 8000 min_disk_gb: 400 archive_pointers: - pointer: !store_id ethereum-reth-sepolia-full-v1 - - pointer: !combination_disallowed - new_archive_properties: [ "tracing-api-on-value" ] firewall_config: default_in: deny default_out: allow @@ -77,19 +69,13 @@ properties: # Property values will be available in a Rhai runtime via `node_params().` (e.g. `node_params().tracing-api`). # It is required to be lower-kebab-case. - key: tracing-api + name: Tracing API description: This enables or disables the tracing API dynamic_value: false - default_value: off-value + default_value: off ui_type: !switch on: - value: on-value - impact: - new_archive: true - add_cpu: 3 - add_memory_bytes: 10000000 - add_disk_bytes: 10000000000 - off: - value: off-value - impact: null - - + add_cpu: 3 + add_memory_bytes: 10000000 + add_disk_bytes: 10000000000 + off: null diff --git a/ethereum/ethereum-reth/main.rhai b/ethereum/ethereum-reth/main.rhai index 813f5faf..ada3c693 100644 --- a/ethereum/ethereum-reth/main.rhai +++ b/ethereum/ethereum-reth/main.rhai @@ -1,7 +1,7 @@ import "base" as base; import "reth" as reth; -const CADDY_DIR = DATA_DRIVE_MOUNT_POINT + "/caddy"; +const CADDY_DIR = node_env().data_mount_point + "/caddy"; const RPC_PORT = 8545; const WS_PORT = 8546; const METRICS_PORT = 6665; @@ -14,26 +14,26 @@ const BASE_CONFIG = #{ }; const VARIANTS = #{ - reth_sepolia_full: #{ + "reth-sepolia-full": #{ net: "sepolia", }, - reth_mainnet_full: #{ + "reth-mainnet-full": #{ net: "mainnet", }, - reth_sepolia_archive: #{ + "reth-sepolia-archive": #{ net: "sepolia", }, - reth_mainnet_archive: #{ + "reth-mainnet-archive": #{ net: "mainnet", }, }; const API_HOST = `http://127.0.0.1:${global::RPC_PORT}`; -const HOME = PROTOCOL_DATA_PATH + "/reth"; +const HOME = node_env().protocol_data_path + "/reth"; const NET = VARIANTS[node_env().node_variant].net; -const CADDY_DIR = DATA_DRIVE_MOUNT_POINT + "/caddy"; +const CADDY_DIR = node_env().data_mount_point + "/caddy"; -const RETH_EXTRA_ARGS = if node_params().nodetype == "full" { +const RETH_EXTRA_ARGS = if node_env().node_variant.ends_with("-full") { "--db-mode=full --pruning.block-limit=1048576" } else { "--db-mode=archive" From 69d65495cb64f3afe6fa37ae1dffff1787900a08 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 19 Dec 2024 13:55:01 +0000 Subject: [PATCH 25/88] Added lighthouse, improved deps --- clients/consensus/lighthouse/Dockerfile | 30 +++++++++++++++++ clients/{ => exec}/erigon/Caddyfile.template | 0 clients/{ => exec}/erigon/Dockerfile | 0 .../{ => exec}/erigon/config-alloy.template | 0 clients/{ => exec}/erigon/erigon.rhai | 0 clients/{ => exec}/reth/Dockerfile | 20 ++---------- .../ethereum-reth}/Caddyfile.template | 0 ethereum/ethereum-reth/Dockerfile | 21 +++++++++++- ethereum/ethereum-reth/babel.yaml | 32 ++----------------- .../ethereum-reth}/config-alloy.template | 0 .../reth => ethereum/ethereum-reth}/reth.rhai | 0 11 files changed, 55 insertions(+), 48 deletions(-) create mode 100644 clients/consensus/lighthouse/Dockerfile rename clients/{ => exec}/erigon/Caddyfile.template (100%) rename clients/{ => exec}/erigon/Dockerfile (100%) rename clients/{ => exec}/erigon/config-alloy.template (100%) rename clients/{ => exec}/erigon/erigon.rhai (100%) rename clients/{ => exec}/reth/Dockerfile (50%) rename {clients/reth => ethereum/ethereum-reth}/Caddyfile.template (100%) rename {clients/reth => ethereum/ethereum-reth}/config-alloy.template (100%) rename {clients/reth => ethereum/ethereum-reth}/reth.rhai (100%) diff --git a/clients/consensus/lighthouse/Dockerfile b/clients/consensus/lighthouse/Dockerfile new file mode 100644 index 00000000..fa7bf8c0 --- /dev/null +++ b/clients/consensus/lighthouse/Dockerfile @@ -0,0 +1,30 @@ +FROM node-base:latest + +ENV DEBIAN_FRONTEND=noninteractive + +ENV LIGHTHOUSE_VERSION=v5.3.0 +ENV CARGO_NET_GIT_FETCH_WITH_CLI=true + +# Install system dependencies including libclang +RUN apt-get update && \ + apt-get install -y \ + clang \ + libclang-dev \ + cmake \ + && rm -rf /var/lib/apt/lists/* + +# Install Rust and required dependencies +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \ + . "$HOME/.cargo/env" && \ + rustup default stable && \ + rustup target add x86_64-unknown-linux-gnu + +# Build Lighthouse +WORKDIR /root +RUN git clone https://github.com/sigp/lighthouse.git && \ + cd lighthouse && \ + git checkout $LIGHTHOUSE_VERSION + +WORKDIR /root/lighthouse +RUN . "$HOME/.cargo/env" && make + diff --git a/clients/erigon/Caddyfile.template b/clients/exec/erigon/Caddyfile.template similarity index 100% rename from clients/erigon/Caddyfile.template rename to clients/exec/erigon/Caddyfile.template diff --git a/clients/erigon/Dockerfile b/clients/exec/erigon/Dockerfile similarity index 100% rename from clients/erigon/Dockerfile rename to clients/exec/erigon/Dockerfile diff --git a/clients/erigon/config-alloy.template b/clients/exec/erigon/config-alloy.template similarity index 100% rename from clients/erigon/config-alloy.template rename to clients/exec/erigon/config-alloy.template diff --git a/clients/erigon/erigon.rhai b/clients/exec/erigon/erigon.rhai similarity index 100% rename from clients/erigon/erigon.rhai rename to clients/exec/erigon/erigon.rhai diff --git a/clients/reth/Dockerfile b/clients/exec/reth/Dockerfile similarity index 50% rename from clients/reth/Dockerfile rename to clients/exec/reth/Dockerfile index 22776b8b..5cb4ab9f 100644 --- a/clients/reth/Dockerfile +++ b/clients/exec/reth/Dockerfile @@ -1,15 +1,9 @@ FROM node-base:latest -# Add build arguments for Cloudflare and Grafana push credentials -ARG CLOUDFLARE_API_KEY -ARG GRAFANA_LOKI_BASICAUTH -ARG GRAFANA_PROM_BASICAUTH ENV DEBIAN_FRONTEND=noninteractive -ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} -ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} -ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} -ENV RETH_VERSION=v1.1.2 +ENV RETH_VERSION=v1.1.4 +ENV LIGHTHOUSE_VERSION=v5.3.0 ENV CARGO_NET_GIT_FETCH_WITH_CLI=true # Install system dependencies including libclang @@ -32,12 +26,4 @@ RUN git clone https://github.com/paradigmxyz/reth.git && \ # Build reth with performance features WORKDIR /root/reth -RUN . "$HOME/.cargo/env" && \ - mkdir /root/bin && \ - cargo build --profile maxperf && \ - cp target/maxperf/reth /root/bin/ && \ - rm -rf /root/reth - -COPY Caddyfile.template /var/lib/babel/templates/Caddyfile.template -COPY config-alloy.template /var/lib/babel/templates/config-alloy.template -COPY reth.rhai /var/lib/babel/plugin/reth.rhai +RUN . "$HOME/.cargo/env" && cargo build --profile maxperf diff --git a/clients/reth/Caddyfile.template b/ethereum/ethereum-reth/Caddyfile.template similarity index 100% rename from clients/reth/Caddyfile.template rename to ethereum/ethereum-reth/Caddyfile.template diff --git a/ethereum/ethereum-reth/Dockerfile b/ethereum/ethereum-reth/Dockerfile index 5c5f89b8..3e1ab011 100644 --- a/ethereum/ethereum-reth/Dockerfile +++ b/ethereum/ethereum-reth/Dockerfile @@ -1,3 +1,22 @@ -FROM reth:latest +FROM reth:latest AS reth-build +FROM lighthouse:latest AS lighthouse-build + +FROM node-base:latest + +ARG CLOUDFLARE_API_KEY +ARG GRAFANA_LOKI_BASICAUTH +ARG GRAFANA_PROM_BASICAUTH + +ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} +ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} +ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} + +RUN mkdir -p /root/bin +COPY --from=reth-build /root/reth/target/maxperf/reth /root/bin/ +COPY --from=lighthouse-build /root/lighthouse/target/release/lighthouse /root/bin/ + +COPY ./Caddyfile.template /var/lib/babel/templates/ +COPY ./reth.rhai /var/lib/babel/plugin/reth.rhai COPY ./main.rhai /var/lib/babel/plugin/ +COPY ./config-alloy.template /var/lib/babel/templates/config-alloy.template \ No newline at end of file diff --git a/ethereum/ethereum-reth/babel.yaml b/ethereum/ethereum-reth/babel.yaml index 37f12890..6592d166 100644 --- a/ethereum/ethereum-reth/babel.yaml +++ b/ethereum/ethereum-reth/babel.yaml @@ -1,4 +1,4 @@ -version: 0.0.1 +version: 0.0.2 container_uri: docker-daemon://ethereum-reth:latest sku_code: ETH-RETH org_id: null @@ -6,21 +6,6 @@ description: Ethereum Reth node visibility: public protocol_key: ethereum variants: - # The same protocol can have different implementations. This field let identify what implementation(s) has been used. - # Variant name may additional contain protocol network name e.g. "eth-mainnet". - # This value will be available in a Rhai runtime via `node_env().variant_key`. - - key: reth-mainnet-archive - min_cpu: 12 - min_memory_mb: 32000 - min_disk_gb: 3500 - archive_pointers: - - pointer: !store_id ethereum-reth-mainnet-archive-v1 - - key: reth-sepolia-archive - min_cpu: 6 - min_memory_mb: 16000 - min_disk_gb: 1000 - archive_pointers: - - pointer: !store_id ethereum-reth-sepolia-archive-v1 - key: reth-mainnet-full min_cpu: 8 min_memory_mb: 16000 @@ -65,17 +50,4 @@ firewall_config: name: P2P port - port: 42069 name: Torrent port -properties: - # Property values will be available in a Rhai runtime via `node_params().` (e.g. `node_params().tracing-api`). - # It is required to be lower-kebab-case. - - key: tracing-api - name: Tracing API - description: This enables or disables the tracing API - dynamic_value: false - default_value: off - ui_type: !switch - on: - add_cpu: 3 - add_memory_bytes: 10000000 - add_disk_bytes: 10000000000 - off: null +properties: [] diff --git a/clients/reth/config-alloy.template b/ethereum/ethereum-reth/config-alloy.template similarity index 100% rename from clients/reth/config-alloy.template rename to ethereum/ethereum-reth/config-alloy.template diff --git a/clients/reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai similarity index 100% rename from clients/reth/reth.rhai rename to ethereum/ethereum-reth/reth.rhai From 81ff549ec77239661c79c61a888e0fd3ae261b1c Mon Sep 17 00:00:00 2001 From: root Date: Thu, 19 Dec 2024 14:25:05 +0000 Subject: [PATCH 26/88] Fixes --- clients/consensus/lighthouse/Dockerfile | 2 +- ethereum/ethereum-reth/Dockerfile | 2 +- ethereum/ethereum-reth/main.rhai | 20 +++++++++++++++++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/clients/consensus/lighthouse/Dockerfile b/clients/consensus/lighthouse/Dockerfile index fa7bf8c0..4b501abe 100644 --- a/clients/consensus/lighthouse/Dockerfile +++ b/clients/consensus/lighthouse/Dockerfile @@ -26,5 +26,5 @@ RUN git clone https://github.com/sigp/lighthouse.git && \ git checkout $LIGHTHOUSE_VERSION WORKDIR /root/lighthouse -RUN . "$HOME/.cargo/env" && make +RUN . "$HOME/.cargo/env" && cargo build diff --git a/ethereum/ethereum-reth/Dockerfile b/ethereum/ethereum-reth/Dockerfile index 3e1ab011..ddb62fe3 100644 --- a/ethereum/ethereum-reth/Dockerfile +++ b/ethereum/ethereum-reth/Dockerfile @@ -17,6 +17,6 @@ COPY --from=reth-build /root/reth/target/maxperf/reth /root/bin/ COPY --from=lighthouse-build /root/lighthouse/target/release/lighthouse /root/bin/ COPY ./Caddyfile.template /var/lib/babel/templates/ -COPY ./reth.rhai /var/lib/babel/plugin/reth.rhai +COPY ./reth.rhai /var/lib/babel/plugin/ COPY ./main.rhai /var/lib/babel/plugin/ COPY ./config-alloy.template /var/lib/babel/templates/config-alloy.template \ No newline at end of file diff --git a/ethereum/ethereum-reth/main.rhai b/ethereum/ethereum-reth/main.rhai index ada3c693..8e5700b0 100644 --- a/ethereum/ethereum-reth/main.rhai +++ b/ethereum/ethereum-reth/main.rhai @@ -73,7 +73,26 @@ const PLUGIN_CONFIG = #{ ${global::RETH_EXTRA_ARGS}`, shutdown_timeout_secs: 120, use_blockchain_data: true, + log_timestamp: false, }, + #{ + name: "lighthouse", + run_sh: `RUST_LOG=info /root/bin/lighthouse bn \ + --network ${global::NET.networkname} \ + --datadir ${global::LIGHTHOUSE_DIR} \ + --checkpoint-sync-url ${global::NET.url} \ + --disable-deposit-contract-sync --http \ + --http-address 127.0.0.1 --http-port 5052 \ + --execution-endpoint http://127.0.0.1:8551 \ + --execution-jwt ${global::HOME}/jwt.hex \ + --metrics --metrics-port 9002 --prune-blobs false \ + --reconstruct-historic-states --block-cache-size 64 \ + --genesis-backfill --historic-state-cache-size 4 \ + --disable-backfill-rate-limiting`, + shutdown_timeout_secs: 120, + use_blockchain_data: true, + log_timestamp: false, + }, ], alternative_download: #{ run_sh: "echo dummy" @@ -96,7 +115,6 @@ fn application_status() { } } - fn height() { parse_hex(run_jrpc(#{host: global::API_HOST, method: "eth_blockNumber"}).expect(200).result) } From c458c91fcce8e4cd13608ce3ec505ebaf5a156de Mon Sep 17 00:00:00 2001 From: root Date: Thu, 19 Dec 2024 14:50:07 +0000 Subject: [PATCH 27/88] RHAI fixes --- ethereum/ethereum-reth/main.rhai | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/ethereum/ethereum-reth/main.rhai b/ethereum/ethereum-reth/main.rhai index 8e5700b0..399ae1b5 100644 --- a/ethereum/ethereum-reth/main.rhai +++ b/ethereum/ethereum-reth/main.rhai @@ -16,27 +16,33 @@ const BASE_CONFIG = #{ const VARIANTS = #{ "reth-sepolia-full": #{ net: "sepolia", + url: "https://sepolia.beaconstate.info", }, "reth-mainnet-full": #{ net: "mainnet", + url: "https://mainnet.checkpoint.sigp.io", }, "reth-sepolia-archive": #{ net: "sepolia", + url: "https://sepolia.beaconstate.info", }, "reth-mainnet-archive": #{ net: "mainnet", + url: "https://mainnet.checkpoint.sigp.io", }, }; const API_HOST = `http://127.0.0.1:${global::RPC_PORT}`; const HOME = node_env().protocol_data_path + "/reth"; +const LIGHTHOUSE_DIR = node_env().protocol_data_path + "/lighthouse"; const NET = VARIANTS[node_env().node_variant].net; const CADDY_DIR = node_env().data_mount_point + "/caddy"; +const CHECKPOINT_URL = VARIANTS[node_env().node_variant].url; const RETH_EXTRA_ARGS = if node_env().node_variant.ends_with("-full") { - "--db-mode=full --pruning.block-limit=1048576" + "--full" } else { - "--db-mode=archive" + "" }; const BABEL_VERSION = "0.9.0"; @@ -45,6 +51,7 @@ const PLUGIN_CONFIG = #{ init: #{ commands: [ `mkdir -p ${global::HOME}`, + `mkdir -p ${global::LIGHTHOUSE_DIR}`, `mkdir -p ${global::CADDY_DIR}`, ], jobs: [ @@ -78,9 +85,9 @@ const PLUGIN_CONFIG = #{ #{ name: "lighthouse", run_sh: `RUST_LOG=info /root/bin/lighthouse bn \ - --network ${global::NET.networkname} \ + --network ${global::NET} \ --datadir ${global::LIGHTHOUSE_DIR} \ - --checkpoint-sync-url ${global::NET.url} \ + --checkpoint-sync-url ${global::CHECKPOINT_URL} \ --disable-deposit-contract-sync --http \ --http-address 127.0.0.1 --http-port 5052 \ --execution-endpoint http://127.0.0.1:8551 \ From 48b7224d5827433a28db55d7dcd6ee8677ba655b Mon Sep 17 00:00:00 2001 From: root Date: Fri, 20 Dec 2024 12:38:05 +0000 Subject: [PATCH 28/88] Updated erigon --- clients/consensus/lighthouse/Dockerfile | 5 +- clients/exec/erigon/Dockerfile | 8 +-- clients/exec/reth/Dockerfile | 4 +- .../ethereum-erigon}/Caddyfile.template | 0 ethereum/ethereum-erigon/Dockerfile | 23 +++++- ethereum/ethereum-erigon/babel.yaml | 70 ++++--------------- .../ethereum-erigon}/config-alloy.template | 0 .../ethereum-erigon}/erigon.rhai | 0 ethereum/ethereum-reth/Dockerfile | 12 ++-- ethereum/ethereum-reth/babel.yaml | 18 ++++- 10 files changed, 65 insertions(+), 75 deletions(-) rename {clients/exec/erigon => ethereum/ethereum-erigon}/Caddyfile.template (100%) rename {clients/exec/erigon => ethereum/ethereum-erigon}/config-alloy.template (100%) rename {clients/exec/erigon => ethereum/ethereum-erigon}/erigon.rhai (100%) diff --git a/clients/consensus/lighthouse/Dockerfile b/clients/consensus/lighthouse/Dockerfile index 4b501abe..d9cff401 100644 --- a/clients/consensus/lighthouse/Dockerfile +++ b/clients/consensus/lighthouse/Dockerfile @@ -11,6 +11,7 @@ RUN apt-get update && \ clang \ libclang-dev \ cmake \ + libpq-dev \ && rm -rf /var/lib/apt/lists/* # Install Rust and required dependencies @@ -26,5 +27,7 @@ RUN git clone https://github.com/sigp/lighthouse.git && \ git checkout $LIGHTHOUSE_VERSION WORKDIR /root/lighthouse -RUN . "$HOME/.cargo/env" && cargo build +RUN . "$HOME/.cargo/env" && cargo build --release --features portable && \ + mkdir -p /root/bin && \ + cp /root/lighthouse/target/release/lighthouse /root/bin/lighthouse diff --git a/clients/exec/erigon/Dockerfile b/clients/exec/erigon/Dockerfile index 3134028c..fa3426bc 100644 --- a/clients/exec/erigon/Dockerfile +++ b/clients/exec/erigon/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/blockjoy/node-base:866a6bb@sha256:7923a8832df1119faf8d56cb501764ad9edb5bba6f54a2382edd0b8a2385bda3 +FROM node-base:latest # Add build arguments for Cloudflare and Grafana push credentials ARG CLOUDFLARE_API_KEY @@ -25,9 +25,5 @@ RUN git clone https://github.com/erigontech/erigon.git && \ WORKDIR /root/erigon RUN make erigon -RUN cp /root/erigon/build/bin/erigon /root/bin/erigon && \ - rm -rf /root/erigon +RUN cp /root/erigon/build/bin/erigon /root/bin/erigon -COPY Caddyfile.template /var/lib/babel/templates/Caddyfile.template -COPY config-alloy.template /var/lib/babel/templates/config-alloy.template -COPY erigon.rhai /var/lib/babel/plugin/erigon.rhai diff --git a/clients/exec/reth/Dockerfile b/clients/exec/reth/Dockerfile index 5cb4ab9f..56f94be9 100644 --- a/clients/exec/reth/Dockerfile +++ b/clients/exec/reth/Dockerfile @@ -26,4 +26,6 @@ RUN git clone https://github.com/paradigmxyz/reth.git && \ # Build reth with performance features WORKDIR /root/reth -RUN . "$HOME/.cargo/env" && cargo build --profile maxperf +RUN . "$HOME/.cargo/env" && cargo build --profile maxperf && \ + mkdir -p /root/bin && \ + cp /root/reth/target/maxperf/reth /root/bin/reth diff --git a/clients/exec/erigon/Caddyfile.template b/ethereum/ethereum-erigon/Caddyfile.template similarity index 100% rename from clients/exec/erigon/Caddyfile.template rename to ethereum/ethereum-erigon/Caddyfile.template diff --git a/ethereum/ethereum-erigon/Dockerfile b/ethereum/ethereum-erigon/Dockerfile index be6a4291..7a10a258 100644 --- a/ethereum/ethereum-erigon/Dockerfile +++ b/ethereum/ethereum-erigon/Dockerfile @@ -1,3 +1,22 @@ -FROM ghcr.io/blockjoy/blockjoy-erigon:2.60.9-e1fce25@sha256:d02854883030ac8555bfc4529668ab7e842a2225b0806cee97128ecbdfa22d3b +FROM ethereum-erigon:latest as erigon-build -COPY ./main.rhai /var/lib/babel/plugin/ +FROM lighthouse:latest AS lighthouse-build + +FROM node-base:latest + +ARG CLOUDFLARE_API_KEY +ARG GRAFANA_LOKI_BASICAUTH +ARG GRAFANA_PROM_BASICAUTH + +ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} +ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} +ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} + +RUN mkdir -p /root/bin +COPY --from=erigon-build /root/bin/erigon /root/bin/ +COPY --from=lighthouse-build /root/bin/lighthouse /root/bin/ + +COPY Caddyfile.template /var/lib/babel/templates/Caddyfile.template +COPY erigon.rhai /var/lib/babel/plugin/ +COPY main.rhai /var/lib/babel/plugin/ +COPY config-alloy.template /var/lib/babel/templates/config-alloy.template diff --git a/ethereum/ethereum-erigon/babel.yaml b/ethereum/ethereum-erigon/babel.yaml index 9f1f89a6..e761026c 100644 --- a/ethereum/ethereum-erigon/babel.yaml +++ b/ethereum/ethereum-erigon/babel.yaml @@ -6,57 +6,30 @@ description: Ethereum Erigon node visibility: public protocol_key: ethereum variants: - # The same protocol can have different implementations. This field let identify what implementation(s) has been used. - # Variant name may additional contain protocol network name e.g. "eth-mainnet". - # This value will be available in a Rhai runtime via `node_env().variant_key`. - - key: erigon-mainnet-archive - archive_pointers: - - pointer: !store_id ethereum-erigon-mainnet-archive-v1 - - pointer: !store_id ethereum-erigon-mainnet-archive-v1 - new_archive_properties: [ "tracing-api-on-value" ] - min_cpu: 8 - min_memory_mb: 32000 + - key: erigon-mainnet-full + min_cpu: 16 + min_memory_mb: 65536 min_disk_gb: 5000 - - key: erigon-sepolia-archive archive_pointers: - - pointer: !store_id ethereum-erigon-sepolia-archive-v1 - - pointer: !store_id ethereum-erigon-sepolia-archive-v1 - new_archive_properties: [ "tracing-api-on-value" ] + - pointer: !store_id ethereum-erigon-mainnet-full-v1 + - key: erigon-sepolia-full min_cpu: 4 min_memory_mb: 16000 min_disk_gb: 2000 - - key: erigon-goerli-archive archive_pointers: - - pointer: !store_id ethereum-erigon-goerli-archive-v1 - - pointer: !store_id ethereum-erigon-goerli-archive-v1 - new_archive_properties: [ "tracing-api-on-value" ] - min_cpu: 4 - min_memory_mb: 16000 - min_disk_gb: 2000 - - key: erigon-mainnet-full - archive_pointers: - - pointer: !store_id ethereum-erigon-mainnet-full-v1 - - pointer: !combination_disallowed - new_archive_properties: [ "tracing-api-on-value" ] - min_cpu: 8 - min_memory_mb: 32000 + - pointer: !store_id ethereum-erigon-sepolia-full-v1 + - key: erigon-mainnet-archive + min_cpu: 16 + min_memory_mb: 65536 min_disk_gb: 5000 - - key: erigon-sepolia-full archive_pointers: - - pointer: !store_id ethereum-erigon-sepolia-full-v1 - - pointer: !combination_disallowed - new_archive_properties: [ "tracing-api-on-value" ] + - pointer: !store_id ethereum-erigon-mainnet-archive-v1 + - key: erigon-sepolia-archive min_cpu: 4 min_memory_mb: 16000 min_disk_gb: 2000 - - key: erigon-goerli-full archive_pointers: - - pointer: !store_id ethereum-erigon-goerli-full-v1 - - pointer: !combination_disallowed - new_archive_properties: [ "tracing-api-on-value" ] - min_cpu: 4 - min_memory_mb: 16000 - min_disk_gb: 2000 + - pointer: !store_id ethereum-erigon-sepolia-archive-v1 firewall_config: default_in: deny default_out: allow @@ -91,21 +64,4 @@ firewall_config: name: P2P port 2 - port: 42069 name: Torrent port -properties: - - key: tracing-api - description: This enables or disables the tracing API - dynamic_value: false - default_value: off-value - ui_type: !switch - on: - value: on-value - impact: - new_archive: true - add_cpu: 3 - add_memory_bytes: 10000000 - add_disk_bytes: 10000000000 - off: - value: off-value - impact: null - # Unique property key. Property value will be available in a Rhai runtime via `node_params().` (e.g. `node_params().arbitrary_property`). - # It is required to be lower-kebab-case. \ No newline at end of file +properties: [] \ No newline at end of file diff --git a/clients/exec/erigon/config-alloy.template b/ethereum/ethereum-erigon/config-alloy.template similarity index 100% rename from clients/exec/erigon/config-alloy.template rename to ethereum/ethereum-erigon/config-alloy.template diff --git a/clients/exec/erigon/erigon.rhai b/ethereum/ethereum-erigon/erigon.rhai similarity index 100% rename from clients/exec/erigon/erigon.rhai rename to ethereum/ethereum-erigon/erigon.rhai diff --git a/ethereum/ethereum-reth/Dockerfile b/ethereum/ethereum-reth/Dockerfile index ddb62fe3..f16ea9ad 100644 --- a/ethereum/ethereum-reth/Dockerfile +++ b/ethereum/ethereum-reth/Dockerfile @@ -13,10 +13,10 @@ ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} RUN mkdir -p /root/bin -COPY --from=reth-build /root/reth/target/maxperf/reth /root/bin/ -COPY --from=lighthouse-build /root/lighthouse/target/release/lighthouse /root/bin/ +COPY --from=reth-build /root/bin/reth /root/bin/ +COPY --from=lighthouse-build /root/bin/lighthouse /root/bin/ -COPY ./Caddyfile.template /var/lib/babel/templates/ -COPY ./reth.rhai /var/lib/babel/plugin/ -COPY ./main.rhai /var/lib/babel/plugin/ -COPY ./config-alloy.template /var/lib/babel/templates/config-alloy.template \ No newline at end of file +COPY Caddyfile.template /var/lib/babel/templates/ +COPY reth.rhai /var/lib/babel/plugin/ +COPY main.rhai /var/lib/babel/plugin/ +COPY config-alloy.template /var/lib/babel/templates/config-alloy.template \ No newline at end of file diff --git a/ethereum/ethereum-reth/babel.yaml b/ethereum/ethereum-reth/babel.yaml index 6592d166..0501bb18 100644 --- a/ethereum/ethereum-reth/babel.yaml +++ b/ethereum/ethereum-reth/babel.yaml @@ -18,6 +18,18 @@ variants: min_disk_gb: 400 archive_pointers: - pointer: !store_id ethereum-reth-sepolia-full-v1 + - key: reth-mainnet-archive + min_cpu: 8 + min_memory_mb: 16000 + min_disk_gb: 4000 + archive_pointers: + - pointer: !store_id ethereum-reth-mainnet-archive-v1 + - key: reth-sepolia-archive + min_cpu: 4 + min_memory_mb: 8000 + min_disk_gb: 800 + archive_pointers: + - pointer: !store_id ethereum-reth-sepolia-archive-v1 firewall_config: default_in: deny default_out: allow @@ -48,6 +60,8 @@ firewall_config: ports: - port: 30303 name: P2P port - - port: 42069 - name: Torrent port + - port: 9000 + name: Metrics port + - port: 9001 + name: Metrics port properties: [] From ad60f7870ae9aba5012cd3dde085e077aec6043a Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 10:41:44 +0200 Subject: [PATCH 29/88] CICD improvements --- .github/workflows/docker-build.yml | 56 ++++++++++++++++++++----- clients/consensus/lighthouse/Dockerfile | 5 ++- clients/exec/erigon/Dockerfile | 5 ++- clients/exec/reth/Dockerfile | 8 ++-- ethereum/ethereum-erigon/Dockerfile | 15 ++++--- ethereum/ethereum-reth/Dockerfile | 14 ++++--- node-base/Dockerfile | 4 +- 7 files changed, 77 insertions(+), 30 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index fe06b63d..f6c005d0 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -74,6 +74,20 @@ jobs: SHA=$(git rev-parse --short HEAD) echo "image_name=node-base" >> $GITHUB_OUTPUT echo "image_tag=${SHA}" >> $GITHUB_OUTPUT + - name: Generate build arguments + id: build-args + run: | + # Extract all ARG declarations that end with _IMAGE from Dockerfile + BUILD_ARGS=$(grep "^ARG.*_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do + # Extract the variable name (everything before =) + var_name=$(echo "$line" | sed 's/ARG \(.*\)=.*/\1/') + # Construct the full image path with version + echo "${var_name}=ghcr.io/blockjoy/${var_name/_IMAGE/}:${{ steps.version.outputs.image_tag }}" + done | tr '\n' ',' | sed 's/,$//') + + # Add the standard build args + BUILD_ARGS="${BUILD_ARGS},GRAFANA_LOKI_API_KEY=${{ secrets.GRAFANA_LOKI_API_KEY }},GRAFANA_PROM_API_KEY=${{ secrets.GRAFANA_PROM_API_KEY }}" + echo "args=${BUILD_ARGS}" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push base image @@ -81,9 +95,7 @@ jobs: with: context: ./${{ matrix.image_path }} push: true - build-args: | - GRAFANA_LOKI_API_KEY=${{ secrets.GRAFANA_LOKI_API_KEY }} - GRAFANA_PROM_API_KEY=${{ secrets.GRAFANA_PROM_API_KEY }} + build-args: ${{ steps.build-args.outputs.args }} cache-from: type=gha cache-to: type=gha,mode=max tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} @@ -125,6 +137,20 @@ jobs: fi echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT echo "image_tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT + - name: Generate build arguments + id: build-args + run: | + # Extract all ARG declarations that end with _IMAGE from Dockerfile + BUILD_ARGS=$(grep "^ARG.*_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do + # Extract the variable name (everything before =) + var_name=$(echo "$line" | sed 's/ARG \(.*\)=.*/\1/') + # Construct the full image path with version + echo "${var_name}=ghcr.io/blockjoy/${var_name/_IMAGE/}:${{ steps.version.outputs.image_tag }}" + done | tr '\n' ',' | sed 's/,$//') + + # Add the standard build args + BUILD_ARGS="${BUILD_ARGS},GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }},GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }},CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }}" + echo "args=${BUILD_ARGS}" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push client images @@ -132,10 +158,7 @@ jobs: with: context: ./${{ matrix.image_path }} push: true - build-args: | - GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }} - GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }} - CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }} + build-args: ${{ steps.build-args.outputs.args }} cache-from: type=gha cache-to: type=gha,mode=max tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} @@ -164,6 +187,20 @@ jobs: IMAGE_NAME=$(basename $(dirname ${{ matrix.image_path }}))"-"$(basename ${{ matrix.image_path }}) echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT echo "image_tag=${SHA}" >> $GITHUB_OUTPUT + - name: Generate build arguments + id: build-args + run: | + # Extract all ARG declarations that end with _IMAGE from Dockerfile + BUILD_ARGS=$(grep "^ARG.*_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do + # Extract the variable name (everything before =) + var_name=$(echo "$line" | sed 's/ARG \(.*\)=.*/\1/') + # Construct the full image path with version + echo "${var_name}=ghcr.io/blockjoy/${var_name/_IMAGE/}:${{ steps.version.outputs.image_tag }}" + done | tr '\n' ',' | sed 's/,$//') + + # Add the standard build args + BUILD_ARGS="${BUILD_ARGS},GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }},GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }},CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }}" + echo "args=${BUILD_ARGS}" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push protocol images @@ -171,10 +208,7 @@ jobs: with: context: ./${{ matrix.image_path }} push: true - build-args: | - GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }} - GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }} - CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }} + build-args: ${{ steps.build-args.outputs.args }} cache-from: type=gha cache-to: type=gha,mode=max tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} diff --git a/clients/consensus/lighthouse/Dockerfile b/clients/consensus/lighthouse/Dockerfile index d9cff401..a49d3d11 100644 --- a/clients/consensus/lighthouse/Dockerfile +++ b/clients/consensus/lighthouse/Dockerfile @@ -1,4 +1,6 @@ -FROM node-base:latest +ARG BASE_IMAGE=ghcr.io/blockjoy/node-base:57f9ec4 + +FROM ${BASE_IMAGE} ENV DEBIAN_FRONTEND=noninteractive @@ -30,4 +32,3 @@ WORKDIR /root/lighthouse RUN . "$HOME/.cargo/env" && cargo build --release --features portable && \ mkdir -p /root/bin && \ cp /root/lighthouse/target/release/lighthouse /root/bin/lighthouse - diff --git a/clients/exec/erigon/Dockerfile b/clients/exec/erigon/Dockerfile index fa3426bc..ed57313e 100644 --- a/clients/exec/erigon/Dockerfile +++ b/clients/exec/erigon/Dockerfile @@ -1,10 +1,12 @@ -FROM node-base:latest +ARG BASE_IMAGE=ghcr.io/blockjoy/node-base:57f9ec4 # Add build arguments for Cloudflare and Grafana push credentials ARG CLOUDFLARE_API_KEY ARG GRAFANA_LOKI_BASICAUTH ARG GRAFANA_PROM_BASICAUTH +FROM ${BASE_IMAGE} + ENV DEBIAN_FRONTEND=noninteractive ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} @@ -26,4 +28,3 @@ WORKDIR /root/erigon RUN make erigon RUN cp /root/erigon/build/bin/erigon /root/bin/erigon - diff --git a/clients/exec/reth/Dockerfile b/clients/exec/reth/Dockerfile index 56f94be9..276a6ffe 100644 --- a/clients/exec/reth/Dockerfile +++ b/clients/exec/reth/Dockerfile @@ -1,9 +1,11 @@ -FROM node-base:latest +ARG BASE_IMAGE=ghcr.io/blockjoy/node-base:57f9ec4 +ARG RETH_VERSION=v1.1.4 +ARG LIGHTHOUSE_VERSION=v5.3.0 + +FROM ${BASE_IMAGE} ENV DEBIAN_FRONTEND=noninteractive -ENV RETH_VERSION=v1.1.4 -ENV LIGHTHOUSE_VERSION=v5.3.0 ENV CARGO_NET_GIT_FETCH_WITH_CLI=true # Install system dependencies including libclang diff --git a/ethereum/ethereum-erigon/Dockerfile b/ethereum/ethereum-erigon/Dockerfile index 7a10a258..e8c294c4 100644 --- a/ethereum/ethereum-erigon/Dockerfile +++ b/ethereum/ethereum-erigon/Dockerfile @@ -1,9 +1,6 @@ -FROM ethereum-erigon:latest as erigon-build - -FROM lighthouse:latest AS lighthouse-build - -FROM node-base:latest - +ARG ERIGON_IMAGE=erigon:latest +ARG LIGHTHOUSE_IMAGE=lighthouse:latest +ARG BASE_IMAGE=node-base:latest ARG CLOUDFLARE_API_KEY ARG GRAFANA_LOKI_BASICAUTH ARG GRAFANA_PROM_BASICAUTH @@ -12,6 +9,12 @@ ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} +FROM ${ERIGON_IMAGE} as erigon-build + +FROM ${LIGHTHOUSE_IMAGE} AS lighthouse-build + +FROM ${BASE_IMAGE} + RUN mkdir -p /root/bin COPY --from=erigon-build /root/bin/erigon /root/bin/ COPY --from=lighthouse-build /root/bin/lighthouse /root/bin/ diff --git a/ethereum/ethereum-reth/Dockerfile b/ethereum/ethereum-reth/Dockerfile index f16ea9ad..c4a38f70 100644 --- a/ethereum/ethereum-reth/Dockerfile +++ b/ethereum/ethereum-reth/Dockerfile @@ -1,8 +1,6 @@ -FROM reth:latest AS reth-build - -FROM lighthouse:latest AS lighthouse-build - -FROM node-base:latest +ARG RETH_IMAGE=reth:latest +ARG LIGHTHOUSE_IMAGE=lighthouse:latest +ARG BASE_IMAGE=node-base:latest ARG CLOUDFLARE_API_KEY ARG GRAFANA_LOKI_BASICAUTH @@ -12,6 +10,12 @@ ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} +FROM ${RETH_IMAGE} AS reth-build + +FROM ${LIGHTHOUSE_IMAGE} AS lighthouse-build + +FROM ${BASE_IMAGE} + RUN mkdir -p /root/bin COPY --from=reth-build /root/bin/reth /root/bin/ COPY --from=lighthouse-build /root/bin/lighthouse /root/bin/ diff --git a/node-base/Dockerfile b/node-base/Dockerfile index 112bb9d0..65a2136c 100644 --- a/node-base/Dockerfile +++ b/node-base/Dockerfile @@ -1,4 +1,6 @@ -FROM debian:bookworm-slim@sha256:a165446a88794db4fec31e35e9441433f9552ae048fb1ed26df352d2b537cb96 +ARG BASE_IMAGE=debian:bookworm-slim@sha256:a165446a88794db4fec31e35e9441433f9552ae048fb1ed26df352d2b537cb96 + +FROM ${BASE_IMAGE} ARG GRAFANA_LOKI_API_KEY ARG GRAFANA_PROM_API_KEY From 78ce4825ced12bcc6de534c3fa70654ac6e28518 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 10:47:15 +0200 Subject: [PATCH 30/88] cicd fixes --- .github/workflows/docker-build.yml | 44 ++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index f6c005d0..3ff46f8c 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -141,16 +141,24 @@ jobs: id: build-args run: | # Extract all ARG declarations that end with _IMAGE from Dockerfile - BUILD_ARGS=$(grep "^ARG.*_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do - # Extract the variable name (everything before =) + IMAGE_ARGS=$(grep "^ARG.*_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do + # Extract the variable name and default value var_name=$(echo "$line" | sed 's/ARG \(.*\)=.*/\1/') - # Construct the full image path with version echo "${var_name}=ghcr.io/blockjoy/${var_name/_IMAGE/}:${{ steps.version.outputs.image_tag }}" - done | tr '\n' ',' | sed 's/,$//') + done) - # Add the standard build args - BUILD_ARGS="${BUILD_ARGS},GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }},GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }},CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }}" - echo "args=${BUILD_ARGS}" >> $GITHUB_OUTPUT + # Format build args as a multiline string + { + echo "$IMAGE_ARGS" + echo "GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }}" + echo "GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }}" + echo "CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }}" + } > build_args.txt + + # Set the output as a multiline string + echo "args<> $GITHUB_OUTPUT + cat build_args.txt >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push client images @@ -191,16 +199,24 @@ jobs: id: build-args run: | # Extract all ARG declarations that end with _IMAGE from Dockerfile - BUILD_ARGS=$(grep "^ARG.*_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do - # Extract the variable name (everything before =) + IMAGE_ARGS=$(grep "^ARG.*_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do + # Extract the variable name and default value var_name=$(echo "$line" | sed 's/ARG \(.*\)=.*/\1/') - # Construct the full image path with version echo "${var_name}=ghcr.io/blockjoy/${var_name/_IMAGE/}:${{ steps.version.outputs.image_tag }}" - done | tr '\n' ',' | sed 's/,$//') + done) - # Add the standard build args - BUILD_ARGS="${BUILD_ARGS},GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }},GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }},CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }}" - echo "args=${BUILD_ARGS}" >> $GITHUB_OUTPUT + # Format build args as a multiline string + { + echo "$IMAGE_ARGS" + echo "GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }}" + echo "GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }}" + echo "CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }}" + } > build_args.txt + + # Set the output as a multiline string + echo "args<> $GITHUB_OUTPUT + cat build_args.txt >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push protocol images From 86928e9a228a6b0d5d2295252bc8561d8119c47d Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 10:53:09 +0200 Subject: [PATCH 31/88] Reverted --- .github/workflows/docker-build.yml | 72 ++++--------------------- clients/consensus/lighthouse/Dockerfile | 5 +- clients/exec/erigon/Dockerfile | 5 +- clients/exec/reth/Dockerfile | 8 ++- ethereum/ethereum-erigon/Dockerfile | 15 +++--- ethereum/ethereum-reth/Dockerfile | 14 ++--- node-base/Dockerfile | 4 +- 7 files changed, 30 insertions(+), 93 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 3ff46f8c..fe06b63d 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -74,20 +74,6 @@ jobs: SHA=$(git rev-parse --short HEAD) echo "image_name=node-base" >> $GITHUB_OUTPUT echo "image_tag=${SHA}" >> $GITHUB_OUTPUT - - name: Generate build arguments - id: build-args - run: | - # Extract all ARG declarations that end with _IMAGE from Dockerfile - BUILD_ARGS=$(grep "^ARG.*_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do - # Extract the variable name (everything before =) - var_name=$(echo "$line" | sed 's/ARG \(.*\)=.*/\1/') - # Construct the full image path with version - echo "${var_name}=ghcr.io/blockjoy/${var_name/_IMAGE/}:${{ steps.version.outputs.image_tag }}" - done | tr '\n' ',' | sed 's/,$//') - - # Add the standard build args - BUILD_ARGS="${BUILD_ARGS},GRAFANA_LOKI_API_KEY=${{ secrets.GRAFANA_LOKI_API_KEY }},GRAFANA_PROM_API_KEY=${{ secrets.GRAFANA_PROM_API_KEY }}" - echo "args=${BUILD_ARGS}" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push base image @@ -95,7 +81,9 @@ jobs: with: context: ./${{ matrix.image_path }} push: true - build-args: ${{ steps.build-args.outputs.args }} + build-args: | + GRAFANA_LOKI_API_KEY=${{ secrets.GRAFANA_LOKI_API_KEY }} + GRAFANA_PROM_API_KEY=${{ secrets.GRAFANA_PROM_API_KEY }} cache-from: type=gha cache-to: type=gha,mode=max tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} @@ -137,28 +125,6 @@ jobs: fi echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT echo "image_tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT - - name: Generate build arguments - id: build-args - run: | - # Extract all ARG declarations that end with _IMAGE from Dockerfile - IMAGE_ARGS=$(grep "^ARG.*_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do - # Extract the variable name and default value - var_name=$(echo "$line" | sed 's/ARG \(.*\)=.*/\1/') - echo "${var_name}=ghcr.io/blockjoy/${var_name/_IMAGE/}:${{ steps.version.outputs.image_tag }}" - done) - - # Format build args as a multiline string - { - echo "$IMAGE_ARGS" - echo "GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }}" - echo "GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }}" - echo "CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }}" - } > build_args.txt - - # Set the output as a multiline string - echo "args<> $GITHUB_OUTPUT - cat build_args.txt >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push client images @@ -166,7 +132,10 @@ jobs: with: context: ./${{ matrix.image_path }} push: true - build-args: ${{ steps.build-args.outputs.args }} + build-args: | + GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }} + GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }} + CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }} cache-from: type=gha cache-to: type=gha,mode=max tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} @@ -195,28 +164,6 @@ jobs: IMAGE_NAME=$(basename $(dirname ${{ matrix.image_path }}))"-"$(basename ${{ matrix.image_path }}) echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT echo "image_tag=${SHA}" >> $GITHUB_OUTPUT - - name: Generate build arguments - id: build-args - run: | - # Extract all ARG declarations that end with _IMAGE from Dockerfile - IMAGE_ARGS=$(grep "^ARG.*_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do - # Extract the variable name and default value - var_name=$(echo "$line" | sed 's/ARG \(.*\)=.*/\1/') - echo "${var_name}=ghcr.io/blockjoy/${var_name/_IMAGE/}:${{ steps.version.outputs.image_tag }}" - done) - - # Format build args as a multiline string - { - echo "$IMAGE_ARGS" - echo "GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }}" - echo "GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }}" - echo "CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }}" - } > build_args.txt - - # Set the output as a multiline string - echo "args<> $GITHUB_OUTPUT - cat build_args.txt >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push protocol images @@ -224,7 +171,10 @@ jobs: with: context: ./${{ matrix.image_path }} push: true - build-args: ${{ steps.build-args.outputs.args }} + build-args: | + GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }} + GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }} + CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }} cache-from: type=gha cache-to: type=gha,mode=max tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} diff --git a/clients/consensus/lighthouse/Dockerfile b/clients/consensus/lighthouse/Dockerfile index a49d3d11..d9cff401 100644 --- a/clients/consensus/lighthouse/Dockerfile +++ b/clients/consensus/lighthouse/Dockerfile @@ -1,6 +1,4 @@ -ARG BASE_IMAGE=ghcr.io/blockjoy/node-base:57f9ec4 - -FROM ${BASE_IMAGE} +FROM node-base:latest ENV DEBIAN_FRONTEND=noninteractive @@ -32,3 +30,4 @@ WORKDIR /root/lighthouse RUN . "$HOME/.cargo/env" && cargo build --release --features portable && \ mkdir -p /root/bin && \ cp /root/lighthouse/target/release/lighthouse /root/bin/lighthouse + diff --git a/clients/exec/erigon/Dockerfile b/clients/exec/erigon/Dockerfile index ed57313e..fa3426bc 100644 --- a/clients/exec/erigon/Dockerfile +++ b/clients/exec/erigon/Dockerfile @@ -1,12 +1,10 @@ -ARG BASE_IMAGE=ghcr.io/blockjoy/node-base:57f9ec4 +FROM node-base:latest # Add build arguments for Cloudflare and Grafana push credentials ARG CLOUDFLARE_API_KEY ARG GRAFANA_LOKI_BASICAUTH ARG GRAFANA_PROM_BASICAUTH -FROM ${BASE_IMAGE} - ENV DEBIAN_FRONTEND=noninteractive ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} @@ -28,3 +26,4 @@ WORKDIR /root/erigon RUN make erigon RUN cp /root/erigon/build/bin/erigon /root/bin/erigon + diff --git a/clients/exec/reth/Dockerfile b/clients/exec/reth/Dockerfile index 276a6ffe..56f94be9 100644 --- a/clients/exec/reth/Dockerfile +++ b/clients/exec/reth/Dockerfile @@ -1,11 +1,9 @@ -ARG BASE_IMAGE=ghcr.io/blockjoy/node-base:57f9ec4 -ARG RETH_VERSION=v1.1.4 -ARG LIGHTHOUSE_VERSION=v5.3.0 - -FROM ${BASE_IMAGE} +FROM node-base:latest ENV DEBIAN_FRONTEND=noninteractive +ENV RETH_VERSION=v1.1.4 +ENV LIGHTHOUSE_VERSION=v5.3.0 ENV CARGO_NET_GIT_FETCH_WITH_CLI=true # Install system dependencies including libclang diff --git a/ethereum/ethereum-erigon/Dockerfile b/ethereum/ethereum-erigon/Dockerfile index e8c294c4..7a10a258 100644 --- a/ethereum/ethereum-erigon/Dockerfile +++ b/ethereum/ethereum-erigon/Dockerfile @@ -1,6 +1,9 @@ -ARG ERIGON_IMAGE=erigon:latest -ARG LIGHTHOUSE_IMAGE=lighthouse:latest -ARG BASE_IMAGE=node-base:latest +FROM ethereum-erigon:latest as erigon-build + +FROM lighthouse:latest AS lighthouse-build + +FROM node-base:latest + ARG CLOUDFLARE_API_KEY ARG GRAFANA_LOKI_BASICAUTH ARG GRAFANA_PROM_BASICAUTH @@ -9,12 +12,6 @@ ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} -FROM ${ERIGON_IMAGE} as erigon-build - -FROM ${LIGHTHOUSE_IMAGE} AS lighthouse-build - -FROM ${BASE_IMAGE} - RUN mkdir -p /root/bin COPY --from=erigon-build /root/bin/erigon /root/bin/ COPY --from=lighthouse-build /root/bin/lighthouse /root/bin/ diff --git a/ethereum/ethereum-reth/Dockerfile b/ethereum/ethereum-reth/Dockerfile index c4a38f70..f16ea9ad 100644 --- a/ethereum/ethereum-reth/Dockerfile +++ b/ethereum/ethereum-reth/Dockerfile @@ -1,6 +1,8 @@ -ARG RETH_IMAGE=reth:latest -ARG LIGHTHOUSE_IMAGE=lighthouse:latest -ARG BASE_IMAGE=node-base:latest +FROM reth:latest AS reth-build + +FROM lighthouse:latest AS lighthouse-build + +FROM node-base:latest ARG CLOUDFLARE_API_KEY ARG GRAFANA_LOKI_BASICAUTH @@ -10,12 +12,6 @@ ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} -FROM ${RETH_IMAGE} AS reth-build - -FROM ${LIGHTHOUSE_IMAGE} AS lighthouse-build - -FROM ${BASE_IMAGE} - RUN mkdir -p /root/bin COPY --from=reth-build /root/bin/reth /root/bin/ COPY --from=lighthouse-build /root/bin/lighthouse /root/bin/ diff --git a/node-base/Dockerfile b/node-base/Dockerfile index 65a2136c..112bb9d0 100644 --- a/node-base/Dockerfile +++ b/node-base/Dockerfile @@ -1,6 +1,4 @@ -ARG BASE_IMAGE=debian:bookworm-slim@sha256:a165446a88794db4fec31e35e9441433f9552ae048fb1ed26df352d2b537cb96 - -FROM ${BASE_IMAGE} +FROM debian:bookworm-slim@sha256:a165446a88794db4fec31e35e9441433f9552ae048fb1ed26df352d2b537cb96 ARG GRAFANA_LOKI_API_KEY ARG GRAFANA_PROM_API_KEY From e72e66415abd5b51c18a6afbef3797b91ca6f1b2 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 11:14:51 +0200 Subject: [PATCH 32/88] Added dynamic build-context --- .github/workflows/docker-build.yml | 42 ++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index fe06b63d..6633d9b2 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -74,6 +74,19 @@ jobs: SHA=$(git rev-parse --short HEAD) echo "image_name=node-base" >> $GITHUB_OUTPUT echo "image_tag=${SHA}" >> $GITHUB_OUTPUT + - name: Generate build contexts + id: build-contexts + run: | + # Extract all FROM directives and their aliases from Dockerfile + BUILD_CONTEXTS=$(grep -h "^FROM" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do + # Extract image name and remove any tag/digest + image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) + # Skip if it's a full registry path + if [[ "$image" != *"/"* ]]; then + echo "${image}=ghcr.io/blockjoy/${image}:${{ steps.version.outputs.image_tag }}" + fi + done | sort -u | tr '\n' ',' | sed 's/,$//') + echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push base image @@ -84,6 +97,7 @@ jobs: build-args: | GRAFANA_LOKI_API_KEY=${{ secrets.GRAFANA_LOKI_API_KEY }} GRAFANA_PROM_API_KEY=${{ secrets.GRAFANA_PROM_API_KEY }} + build-contexts: ${{ steps.build-contexts.outputs.contexts }} cache-from: type=gha cache-to: type=gha,mode=max tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} @@ -125,6 +139,19 @@ jobs: fi echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT echo "image_tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT + - name: Generate build contexts + id: build-contexts + run: | + # Extract all FROM directives and their aliases from Dockerfile + BUILD_CONTEXTS=$(grep -h "^FROM" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do + # Extract image name and remove any tag/digest + image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) + # Skip if it's a full registry path + if [[ "$image" != *"/"* ]]; then + echo "${image}=ghcr.io/blockjoy/${image}:${{ steps.version.outputs.image_tag }}" + fi + done | sort -u | tr '\n' ',' | sed 's/,$//') + echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push client images @@ -136,6 +163,7 @@ jobs: GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }} GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }} CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }} + build-contexts: ${{ steps.build-contexts.outputs.contexts }} cache-from: type=gha cache-to: type=gha,mode=max tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} @@ -164,6 +192,19 @@ jobs: IMAGE_NAME=$(basename $(dirname ${{ matrix.image_path }}))"-"$(basename ${{ matrix.image_path }}) echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT echo "image_tag=${SHA}" >> $GITHUB_OUTPUT + - name: Generate build contexts + id: build-contexts + run: | + # Extract all FROM directives and their aliases from Dockerfile + BUILD_CONTEXTS=$(grep -h "^FROM" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do + # Extract image name and remove any tag/digest + image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) + # Skip if it's a full registry path + if [[ "$image" != *"/"* ]]; then + echo "${image}=ghcr.io/blockjoy/${image}:${{ steps.version.outputs.image_tag }}" + fi + done | sort -u | tr '\n' ',' | sed 's/,$//') + echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push protocol images @@ -175,6 +216,7 @@ jobs: GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }} GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }} CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }} + build-contexts: ${{ steps.build-contexts.outputs.contexts }} cache-from: type=gha cache-to: type=gha,mode=max tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} From c752c185891658f3f6c7f96544708550537fb3e9 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 11:24:42 +0200 Subject: [PATCH 33/88] Only include our images in build-context --- .github/workflows/docker-build.yml | 48 ++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 6633d9b2..53afb096 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -77,12 +77,22 @@ jobs: - name: Generate build contexts id: build-contexts run: | - # Extract all FROM directives and their aliases from Dockerfile + # First, find all directories containing Dockerfiles to build our valid image list + VALID_IMAGES=$(find . -name Dockerfile -exec dirname {} \; | while read dir; do + # Convert directory path to image name (e.g., ./ethereum/ethereum-erigon -> ethereum-erigon) + basename "$dir" | tr '[:upper:]' '[:lower:]' + # For protocol directories, also add the protocol-client format + if [[ "$dir" =~ ^./[^/]+/[^/]+ ]]; then + echo "$dir" | sed 's|^./\([^/]\+\)/\([^/]\+\)|\1-\2|' | tr '[:upper:]' '[:lower:]' + fi + done | sort -u) + + # Now extract FROM directives and filter against our valid images BUILD_CONTEXTS=$(grep -h "^FROM" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do # Extract image name and remove any tag/digest image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) - # Skip if it's a full registry path - if [[ "$image" != *"/"* ]]; then + # Check if this image is in our valid image list + if echo "$VALID_IMAGES" | grep -q "^${image}$"; then echo "${image}=ghcr.io/blockjoy/${image}:${{ steps.version.outputs.image_tag }}" fi done | sort -u | tr '\n' ',' | sed 's/,$//') @@ -142,12 +152,22 @@ jobs: - name: Generate build contexts id: build-contexts run: | - # Extract all FROM directives and their aliases from Dockerfile + # First, find all directories containing Dockerfiles to build our valid image list + VALID_IMAGES=$(find . -name Dockerfile -exec dirname {} \; | while read dir; do + # Convert directory path to image name (e.g., ./ethereum/ethereum-erigon -> ethereum-erigon) + basename "$dir" | tr '[:upper:]' '[:lower:]' + # For protocol directories, also add the protocol-client format + if [[ "$dir" =~ ^./[^/]+/[^/]+ ]]; then + echo "$dir" | sed 's|^./\([^/]\+\)/\([^/]\+\)|\1-\2|' | tr '[:upper:]' '[:lower:]' + fi + done | sort -u) + + # Now extract FROM directives and filter against our valid images BUILD_CONTEXTS=$(grep -h "^FROM" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do # Extract image name and remove any tag/digest image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) - # Skip if it's a full registry path - if [[ "$image" != *"/"* ]]; then + # Check if this image is in our valid image list + if echo "$VALID_IMAGES" | grep -q "^${image}$"; then echo "${image}=ghcr.io/blockjoy/${image}:${{ steps.version.outputs.image_tag }}" fi done | sort -u | tr '\n' ',' | sed 's/,$//') @@ -195,12 +215,22 @@ jobs: - name: Generate build contexts id: build-contexts run: | - # Extract all FROM directives and their aliases from Dockerfile + # First, find all directories containing Dockerfiles to build our valid image list + VALID_IMAGES=$(find . -name Dockerfile -exec dirname {} \; | while read dir; do + # Convert directory path to image name (e.g., ./ethereum/ethereum-erigon -> ethereum-erigon) + basename "$dir" | tr '[:upper:]' '[:lower:]' + # For protocol directories, also add the protocol-client format + if [[ "$dir" =~ ^./[^/]+/[^/]+ ]]; then + echo "$dir" | sed 's|^./\([^/]\+\)/\([^/]\+\)|\1-\2|' | tr '[:upper:]' '[:lower:]' + fi + done | sort -u) + + # Now extract FROM directives and filter against our valid images BUILD_CONTEXTS=$(grep -h "^FROM" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do # Extract image name and remove any tag/digest image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) - # Skip if it's a full registry path - if [[ "$image" != *"/"* ]]; then + # Check if this image is in our valid image list + if echo "$VALID_IMAGES" | grep -q "^${image}$"; then echo "${image}=ghcr.io/blockjoy/${image}:${{ steps.version.outputs.image_tag }}" fi done | sort -u | tr '\n' ',' | sed 's/,$//') From 7f25713d339341cd3beea34be7a5b6d398e18f59 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 11:30:30 +0200 Subject: [PATCH 34/88] node-base tag fix --- .github/workflows/docker-build.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 53afb096..e0a7aca5 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -87,13 +87,17 @@ jobs: fi done | sort -u) + # Get the SHA for base images + SHA=$(git rev-parse --short HEAD) + # Now extract FROM directives and filter against our valid images BUILD_CONTEXTS=$(grep -h "^FROM" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do # Extract image name and remove any tag/digest image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then - echo "${image}=ghcr.io/blockjoy/${image}:${{ steps.version.outputs.image_tag }}" + # Always use SHA tag for base images + echo "${image}=ghcr.io/blockjoy/${image}:${SHA}" fi done | sort -u | tr '\n' ',' | sed 's/,$//') echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT @@ -162,13 +166,17 @@ jobs: fi done | sort -u) + # Get the SHA for base images + SHA=$(git rev-parse --short HEAD) + # Now extract FROM directives and filter against our valid images BUILD_CONTEXTS=$(grep -h "^FROM" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do # Extract image name and remove any tag/digest image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then - echo "${image}=ghcr.io/blockjoy/${image}:${{ steps.version.outputs.image_tag }}" + # Always use SHA tag for base images + echo "${image}=ghcr.io/blockjoy/${image}:${SHA}" fi done | sort -u | tr '\n' ',' | sed 's/,$//') echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT @@ -225,13 +233,17 @@ jobs: fi done | sort -u) + # Get the SHA for base images + SHA=$(git rev-parse --short HEAD) + # Now extract FROM directives and filter against our valid images BUILD_CONTEXTS=$(grep -h "^FROM" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do # Extract image name and remove any tag/digest image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then - echo "${image}=ghcr.io/blockjoy/${image}:${{ steps.version.outputs.image_tag }}" + # Always use SHA tag for base images + echo "${image}=ghcr.io/blockjoy/${image}:${SHA}" fi done | sort -u | tr '\n' ',' | sed 's/,$//') echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT From db096b3b9a2e6c28b39e9e94ccbb6973a9e1bc18 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 11:33:28 +0200 Subject: [PATCH 35/88] fix client tag --- .github/workflows/docker-build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index e0a7aca5..6760f425 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -143,7 +143,9 @@ jobs: if [[ -f "${{ matrix.image_path }}/Dockerfile" ]]; then CLIENT_VERSION=$(grep -E "ENV.*_VERSION=[[:space:]]*v?[0-9]+\.[0-9]+\.[0-9]+[-.a-zA-Z0-9]*" "${{ matrix.image_path }}/Dockerfile" | grep -oE "v?[0-9]+\.[0-9]+\.[0-9]+[-.a-zA-Z0-9]*") if [[ ! -z "$CLIENT_VERSION" ]]; then + # Remove 'v' prefix and ensure version is Docker-compatible (only alphanumeric and . - _) CLIENT_VERSION=${CLIENT_VERSION#v} + CLIENT_VERSION=$(echo "$CLIENT_VERSION" | tr -cd '[:alnum:].-') IMAGE_TAG="${CLIENT_VERSION}-${SHA}" else IMAGE_TAG="${SHA}" From 478ade0a27f9ad8ea4d72d8a68cebd97c839f502 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 11:40:03 +0200 Subject: [PATCH 36/88] fix node-base tag --- .github/workflows/docker-build.yml | 36 +++++++++++++++--------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 6760f425..efa28b14 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -91,16 +91,16 @@ jobs: SHA=$(git rev-parse --short HEAD) # Now extract FROM directives and filter against our valid images - BUILD_CONTEXTS=$(grep -h "^FROM" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do + BUILD_ARGS=$(grep -h "^FROM" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do # Extract image name and remove any tag/digest image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then - # Always use SHA tag for base images - echo "${image}=ghcr.io/blockjoy/${image}:${SHA}" + # Generate --from argument for each base image + echo "DOCKER_BUILDKIT=1 --from=${image}=ghcr.io/blockjoy/${image}:${SHA}" fi - done | sort -u | tr '\n' ',' | sed 's/,$//') - echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT + done | sort -u | tr '\n' ' ') + echo "args=${BUILD_ARGS}" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push base image @@ -111,7 +111,7 @@ jobs: build-args: | GRAFANA_LOKI_API_KEY=${{ secrets.GRAFANA_LOKI_API_KEY }} GRAFANA_PROM_API_KEY=${{ secrets.GRAFANA_PROM_API_KEY }} - build-contexts: ${{ steps.build-contexts.outputs.contexts }} + ${{ steps.build-contexts.outputs.args }} cache-from: type=gha cache-to: type=gha,mode=max tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} @@ -172,16 +172,16 @@ jobs: SHA=$(git rev-parse --short HEAD) # Now extract FROM directives and filter against our valid images - BUILD_CONTEXTS=$(grep -h "^FROM" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do + BUILD_ARGS=$(grep -h "^FROM" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do # Extract image name and remove any tag/digest image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then - # Always use SHA tag for base images - echo "${image}=ghcr.io/blockjoy/${image}:${SHA}" + # Generate --from argument for each base image + echo "DOCKER_BUILDKIT=1 --from=${image}=ghcr.io/blockjoy/${image}:${SHA}" fi - done | sort -u | tr '\n' ',' | sed 's/,$//') - echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT + done | sort -u | tr '\n' ' ') + echo "args=${BUILD_ARGS}" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push client images @@ -193,7 +193,7 @@ jobs: GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }} GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }} CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }} - build-contexts: ${{ steps.build-contexts.outputs.contexts }} + ${{ steps.build-contexts.outputs.args }} cache-from: type=gha cache-to: type=gha,mode=max tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} @@ -239,16 +239,16 @@ jobs: SHA=$(git rev-parse --short HEAD) # Now extract FROM directives and filter against our valid images - BUILD_CONTEXTS=$(grep -h "^FROM" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do + BUILD_ARGS=$(grep -h "^FROM" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do # Extract image name and remove any tag/digest image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then - # Always use SHA tag for base images - echo "${image}=ghcr.io/blockjoy/${image}:${SHA}" + # Generate --from argument for each base image + echo "DOCKER_BUILDKIT=1 --from=${image}=ghcr.io/blockjoy/${image}:${SHA}" fi - done | sort -u | tr '\n' ',' | sed 's/,$//') - echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT + done | sort -u | tr '\n' ' ') + echo "args=${BUILD_ARGS}" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push protocol images @@ -260,7 +260,7 @@ jobs: GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }} GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }} CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }} - build-contexts: ${{ steps.build-contexts.outputs.contexts }} + ${{ steps.build-contexts.outputs.args }} cache-from: type=gha cache-to: type=gha,mode=max tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} From ee2162bb0ef0d100c7573665294de1d9c94f14f1 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 11:45:18 +0200 Subject: [PATCH 37/88] fix image URI --- .github/workflows/docker-build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index efa28b14..33ffb579 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -96,8 +96,8 @@ jobs: image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then - # Generate --from argument for each base image - echo "DOCKER_BUILDKIT=1 --from=${image}=ghcr.io/blockjoy/${image}:${SHA}" + # Generate --from argument for each base image, using full path in both places + echo "--from=ghcr.io/blockjoy/${image}=ghcr.io/blockjoy/${image}:${SHA}" fi done | sort -u | tr '\n' ' ') echo "args=${BUILD_ARGS}" >> $GITHUB_OUTPUT @@ -177,8 +177,8 @@ jobs: image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then - # Generate --from argument for each base image - echo "DOCKER_BUILDKIT=1 --from=${image}=ghcr.io/blockjoy/${image}:${SHA}" + # Generate --from argument for each base image, using full path in both places + echo "--from=ghcr.io/blockjoy/${image}=ghcr.io/blockjoy/${image}:${SHA}" fi done | sort -u | tr '\n' ' ') echo "args=${BUILD_ARGS}" >> $GITHUB_OUTPUT @@ -244,8 +244,8 @@ jobs: image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then - # Generate --from argument for each base image - echo "DOCKER_BUILDKIT=1 --from=${image}=ghcr.io/blockjoy/${image}:${SHA}" + # Generate --from argument for each base image, using full path in both places + echo "--from=ghcr.io/blockjoy/${image}=ghcr.io/blockjoy/${image}:${SHA}" fi done | sort -u | tr '\n' ' ') echo "args=${BUILD_ARGS}" >> $GITHUB_OUTPUT From a2d716d6057181cb5ec7708f54a354020398b98e Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 11:49:30 +0200 Subject: [PATCH 38/88] Fixed image URI --- .github/workflows/docker-build.yml | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 33ffb579..f2800d20 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -93,11 +93,13 @@ jobs: # Now extract FROM directives and filter against our valid images BUILD_ARGS=$(grep -h "^FROM" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do # Extract image name and remove any tag/digest - image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) + full_image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) + # Extract just the image name from the full path + image=$(basename "$full_image") # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then - # Generate --from argument for each base image, using full path in both places - echo "--from=ghcr.io/blockjoy/${image}=ghcr.io/blockjoy/${image}:${SHA}" + # Generate --from argument for each base image + echo "--from=${full_image}=ghcr.io/blockjoy/${image}:${SHA}" fi done | sort -u | tr '\n' ' ') echo "args=${BUILD_ARGS}" >> $GITHUB_OUTPUT @@ -174,11 +176,13 @@ jobs: # Now extract FROM directives and filter against our valid images BUILD_ARGS=$(grep -h "^FROM" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do # Extract image name and remove any tag/digest - image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) + full_image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) + # Extract just the image name from the full path + image=$(basename "$full_image") # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then - # Generate --from argument for each base image, using full path in both places - echo "--from=ghcr.io/blockjoy/${image}=ghcr.io/blockjoy/${image}:${SHA}" + # Generate --from argument for each base image + echo "--from=${full_image}=ghcr.io/blockjoy/${image}:${SHA}" fi done | sort -u | tr '\n' ' ') echo "args=${BUILD_ARGS}" >> $GITHUB_OUTPUT @@ -241,11 +245,13 @@ jobs: # Now extract FROM directives and filter against our valid images BUILD_ARGS=$(grep -h "^FROM" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do # Extract image name and remove any tag/digest - image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) + full_image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) + # Extract just the image name from the full path + image=$(basename "$full_image") # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then - # Generate --from argument for each base image, using full path in both places - echo "--from=ghcr.io/blockjoy/${image}=ghcr.io/blockjoy/${image}:${SHA}" + # Generate --from argument for each base image + echo "--from=${full_image}=ghcr.io/blockjoy/${image}:${SHA}" fi done | sort -u | tr '\n' ' ') echo "args=${BUILD_ARGS}" >> $GITHUB_OUTPUT From 3e6b9ff33bbff94abdfbcf884bff7c886e307b37 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 11:57:52 +0200 Subject: [PATCH 39/88] Fixed image URIs --- .github/workflows/docker-build.yml | 55 +++++++++++-------------- clients/consensus/lighthouse/Dockerfile | 4 +- clients/exec/erigon/Dockerfile | 4 +- ethereum/ethereum-erigon/Dockerfile | 3 +- 4 files changed, 31 insertions(+), 35 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index f2800d20..ebcac36d 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -91,18 +91,17 @@ jobs: SHA=$(git rev-parse --short HEAD) # Now extract FROM directives and filter against our valid images - BUILD_ARGS=$(grep -h "^FROM" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do - # Extract image name and remove any tag/digest - full_image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) - # Extract just the image name from the full path - image=$(basename "$full_image") + BUILD_CONTEXTS=$(grep -h "^FROM.*\${BASE_IMAGE}" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do + # Extract image name from ARG BASE_IMAGE default value + image=$(grep "^ARG BASE_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | cut -d'=' -f2 | cut -d':' -f1) # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then - # Generate --from argument for each base image - echo "--from=${full_image}=ghcr.io/blockjoy/${image}:${SHA}" + # Use short name for context key but full path for value + echo "${image}=ghcr.io/blockjoy/${image}:${SHA}" fi - done | sort -u | tr '\n' ' ') - echo "args=${BUILD_ARGS}" >> $GITHUB_OUTPUT + done | sort -u | tr '\n' ',' | sed 's/,$//') + echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push base image @@ -113,7 +112,7 @@ jobs: build-args: | GRAFANA_LOKI_API_KEY=${{ secrets.GRAFANA_LOKI_API_KEY }} GRAFANA_PROM_API_KEY=${{ secrets.GRAFANA_PROM_API_KEY }} - ${{ steps.build-contexts.outputs.args }} + build-contexts: ${{ steps.build-contexts.outputs.contexts }} cache-from: type=gha cache-to: type=gha,mode=max tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} @@ -174,18 +173,16 @@ jobs: SHA=$(git rev-parse --short HEAD) # Now extract FROM directives and filter against our valid images - BUILD_ARGS=$(grep -h "^FROM" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do - # Extract image name and remove any tag/digest - full_image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) - # Extract just the image name from the full path - image=$(basename "$full_image") + BUILD_CONTEXTS=$(grep -h "^FROM.*\${BASE_IMAGE}" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do + # Extract image name from ARG BASE_IMAGE default value + image=$(grep "^ARG BASE_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | cut -d'=' -f2 | cut -d':' -f1) # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then - # Generate --from argument for each base image - echo "--from=${full_image}=ghcr.io/blockjoy/${image}:${SHA}" + # Use short name for context key but full path for value + echo "${image}=ghcr.io/blockjoy/${image}:${SHA}" fi - done | sort -u | tr '\n' ' ') - echo "args=${BUILD_ARGS}" >> $GITHUB_OUTPUT + done | sort -u | tr '\n' ',' | sed 's/,$//') + echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push client images @@ -197,7 +194,7 @@ jobs: GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }} GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }} CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }} - ${{ steps.build-contexts.outputs.args }} + build-contexts: ${{ steps.build-contexts.outputs.contexts }} cache-from: type=gha cache-to: type=gha,mode=max tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} @@ -243,18 +240,16 @@ jobs: SHA=$(git rev-parse --short HEAD) # Now extract FROM directives and filter against our valid images - BUILD_ARGS=$(grep -h "^FROM" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do - # Extract image name and remove any tag/digest - full_image=$(echo "$line" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) - # Extract just the image name from the full path - image=$(basename "$full_image") + BUILD_CONTEXTS=$(grep -h "^FROM.*\${BASE_IMAGE}" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do + # Extract image name from ARG BASE_IMAGE default value + image=$(grep "^ARG BASE_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | cut -d'=' -f2 | cut -d':' -f1) # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then - # Generate --from argument for each base image - echo "--from=${full_image}=ghcr.io/blockjoy/${image}:${SHA}" + # Use short name for context key but full path for value + echo "${image}=ghcr.io/blockjoy/${image}:${SHA}" fi - done | sort -u | tr '\n' ' ') - echo "args=${BUILD_ARGS}" >> $GITHUB_OUTPUT + done | sort -u | tr '\n' ',' | sed 's/,$//') + echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push protocol images @@ -266,7 +261,7 @@ jobs: GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }} GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }} CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }} - ${{ steps.build-contexts.outputs.args }} + build-contexts: ${{ steps.build-contexts.outputs.contexts }} cache-from: type=gha cache-to: type=gha,mode=max tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} diff --git a/clients/consensus/lighthouse/Dockerfile b/clients/consensus/lighthouse/Dockerfile index d9cff401..6fed12bb 100644 --- a/clients/consensus/lighthouse/Dockerfile +++ b/clients/consensus/lighthouse/Dockerfile @@ -1,4 +1,5 @@ -FROM node-base:latest +ARG BASE_IMAGE=node-base:latest +FROM ghcr.io/blockjoy/${BASE_IMAGE} ENV DEBIAN_FRONTEND=noninteractive @@ -30,4 +31,3 @@ WORKDIR /root/lighthouse RUN . "$HOME/.cargo/env" && cargo build --release --features portable && \ mkdir -p /root/bin && \ cp /root/lighthouse/target/release/lighthouse /root/bin/lighthouse - diff --git a/clients/exec/erigon/Dockerfile b/clients/exec/erigon/Dockerfile index fa3426bc..8538ae07 100644 --- a/clients/exec/erigon/Dockerfile +++ b/clients/exec/erigon/Dockerfile @@ -1,4 +1,5 @@ -FROM node-base:latest +ARG BASE_IMAGE=node-base:latest +FROM ghcr.io/blockjoy/${BASE_IMAGE} # Add build arguments for Cloudflare and Grafana push credentials ARG CLOUDFLARE_API_KEY @@ -26,4 +27,3 @@ WORKDIR /root/erigon RUN make erigon RUN cp /root/erigon/build/bin/erigon /root/bin/erigon - diff --git a/ethereum/ethereum-erigon/Dockerfile b/ethereum/ethereum-erigon/Dockerfile index 7a10a258..4dd730ba 100644 --- a/ethereum/ethereum-erigon/Dockerfile +++ b/ethereum/ethereum-erigon/Dockerfile @@ -2,7 +2,8 @@ FROM ethereum-erigon:latest as erigon-build FROM lighthouse:latest AS lighthouse-build -FROM node-base:latest +ARG BASE_IMAGE=node-base:latest +FROM ${BASE_IMAGE} ARG CLOUDFLARE_API_KEY ARG GRAFANA_LOKI_BASICAUTH From 9429ead4f94395a72d9bd0a5631eae1b11b4bfb4 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 12:02:55 +0200 Subject: [PATCH 40/88] Fixed image URIs again --- .github/workflows/docker-build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index ebcac36d..ba7c5588 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -96,8 +96,8 @@ jobs: image=$(grep "^ARG BASE_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | cut -d'=' -f2 | cut -d':' -f1) # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then - # Use short name for context key but full path for value - echo "${image}=ghcr.io/blockjoy/${image}:${SHA}" + # Use short name for context key but specify it as a docker-image context + echo "${image}=docker-image://ghcr.io/blockjoy/${image}:${SHA}" fi done | sort -u | tr '\n' ',' | sed 's/,$//') echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT @@ -178,8 +178,8 @@ jobs: image=$(grep "^ARG BASE_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | cut -d'=' -f2 | cut -d':' -f1) # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then - # Use short name for context key but full path for value - echo "${image}=ghcr.io/blockjoy/${image}:${SHA}" + # Use short name for context key but specify it as a docker-image context + echo "${image}=docker-image://ghcr.io/blockjoy/${image}:${SHA}" fi done | sort -u | tr '\n' ',' | sed 's/,$//') echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT @@ -245,8 +245,8 @@ jobs: image=$(grep "^ARG BASE_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | cut -d'=' -f2 | cut -d':' -f1) # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then - # Use short name for context key but full path for value - echo "${image}=ghcr.io/blockjoy/${image}:${SHA}" + # Use short name for context key but specify it as a docker-image context + echo "${image}=docker-image://ghcr.io/blockjoy/${image}:${SHA}" fi done | sort -u | tr '\n' ',' | sed 's/,$//') echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT From 559bd52718665d448d8c5ca6ad5954bf407842ff Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 12:15:35 +0200 Subject: [PATCH 41/88] Fixed workflow --- .github/workflows/docker-build.yml | 34 ++++++++++++++++++++--------- clients/exec/reth/Dockerfile | 3 ++- ethereum/ethereum-erigon/Dockerfile | 14 +++++++----- ethereum/ethereum-reth/Dockerfile | 15 ++++++++----- 4 files changed, 43 insertions(+), 23 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index ba7c5588..ae8af734 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -91,9 +91,11 @@ jobs: SHA=$(git rev-parse --short HEAD) # Now extract FROM directives and filter against our valid images - BUILD_CONTEXTS=$(grep -h "^FROM.*\${BASE_IMAGE}" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do - # Extract image name from ARG BASE_IMAGE default value - image=$(grep "^ARG BASE_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | cut -d'=' -f2 | cut -d':' -f1) + BUILD_CONTEXTS=$(grep -h "^FROM.*\${.*_IMAGE}" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do + # Extract variable name from FROM line + var_name=$(echo "$line" | grep -o '\${[^}]*}' | tr -d '${}\n') + # Extract image name from ARG statement + image=$(grep "^ARG ${var_name}=" ./${{ matrix.image_path }}/Dockerfile | cut -d'=' -f2 | cut -d':' -f1) # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then # Use short name for context key but specify it as a docker-image context @@ -101,7 +103,9 @@ jobs: fi done | sort -u | tr '\n' ',' | sed 's/,$//') echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT - + echo "Generated build contexts: ${BUILD_CONTEXTS}" + echo "Valid images: ${VALID_IMAGES}" + echo "Current image path: ${{ matrix.image_path }}" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push base image @@ -173,9 +177,11 @@ jobs: SHA=$(git rev-parse --short HEAD) # Now extract FROM directives and filter against our valid images - BUILD_CONTEXTS=$(grep -h "^FROM.*\${BASE_IMAGE}" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do - # Extract image name from ARG BASE_IMAGE default value - image=$(grep "^ARG BASE_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | cut -d'=' -f2 | cut -d':' -f1) + BUILD_CONTEXTS=$(grep -h "^FROM.*\${.*_IMAGE}" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do + # Extract variable name from FROM line + var_name=$(echo "$line" | grep -o '\${[^}]*}' | tr -d '${}\n') + # Extract image name from ARG statement + image=$(grep "^ARG ${var_name}=" ./${{ matrix.image_path }}/Dockerfile | cut -d'=' -f2 | cut -d':' -f1) # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then # Use short name for context key but specify it as a docker-image context @@ -183,6 +189,9 @@ jobs: fi done | sort -u | tr '\n' ',' | sed 's/,$//') echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT + echo "Generated build contexts: ${BUILD_CONTEXTS}" + echo "Valid images: ${VALID_IMAGES}" + echo "Current image path: ${{ matrix.image_path }}" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push client images @@ -240,9 +249,11 @@ jobs: SHA=$(git rev-parse --short HEAD) # Now extract FROM directives and filter against our valid images - BUILD_CONTEXTS=$(grep -h "^FROM.*\${BASE_IMAGE}" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do - # Extract image name from ARG BASE_IMAGE default value - image=$(grep "^ARG BASE_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | cut -d'=' -f2 | cut -d':' -f1) + BUILD_CONTEXTS=$(grep -h "^FROM.*\${.*_IMAGE}" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do + # Extract variable name from FROM line + var_name=$(echo "$line" | grep -o '\${[^}]*}' | tr -d '${}\n') + # Extract image name from ARG statement + image=$(grep "^ARG ${var_name}=" ./${{ matrix.image_path }}/Dockerfile | cut -d'=' -f2 | cut -d':' -f1) # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then # Use short name for context key but specify it as a docker-image context @@ -250,6 +261,9 @@ jobs: fi done | sort -u | tr '\n' ',' | sed 's/,$//') echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT + echo "Generated build contexts: ${BUILD_CONTEXTS}" + echo "Valid images: ${VALID_IMAGES}" + echo "Current image path: ${{ matrix.image_path }}" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push protocol images diff --git a/clients/exec/reth/Dockerfile b/clients/exec/reth/Dockerfile index 56f94be9..58f214f4 100644 --- a/clients/exec/reth/Dockerfile +++ b/clients/exec/reth/Dockerfile @@ -1,4 +1,5 @@ -FROM node-base:latest +ARG BASE_IMAGE=node-base:latest +FROM ghcr.io/blockjoy/${BASE_IMAGE} ENV DEBIAN_FRONTEND=noninteractive diff --git a/ethereum/ethereum-erigon/Dockerfile b/ethereum/ethereum-erigon/Dockerfile index 4dd730ba..703f7e1f 100644 --- a/ethereum/ethereum-erigon/Dockerfile +++ b/ethereum/ethereum-erigon/Dockerfile @@ -1,14 +1,16 @@ -FROM ethereum-erigon:latest as erigon-build - -FROM lighthouse:latest AS lighthouse-build - +ARG ERIGON_IMAGE=ethereum-erigon:latest +ARG LIGHTHOUSE_IMAGE=lighthouse:latest ARG BASE_IMAGE=node-base:latest -FROM ${BASE_IMAGE} - ARG CLOUDFLARE_API_KEY ARG GRAFANA_LOKI_BASICAUTH ARG GRAFANA_PROM_BASICAUTH +FROM ghcr.io/blockjoy/${ERIGON_IMAGE} as erigon-build + +FROM ghcr.io/blockjoy/${LIGHTHOUSE_IMAGE} AS lighthouse-build + +FROM ghcr.io/blockjoy/${BASE_IMAGE} + ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} diff --git a/ethereum/ethereum-reth/Dockerfile b/ethereum/ethereum-reth/Dockerfile index f16ea9ad..8bbec420 100644 --- a/ethereum/ethereum-reth/Dockerfile +++ b/ethereum/ethereum-reth/Dockerfile @@ -1,9 +1,6 @@ -FROM reth:latest AS reth-build - -FROM lighthouse:latest AS lighthouse-build - -FROM node-base:latest - +ARG RETH_IMAGE=reth:latest +ARG LIGHTHOUSE_IMAGE=lighthouse:latest +ARG BASE_IMAGE=node-base:latest ARG CLOUDFLARE_API_KEY ARG GRAFANA_LOKI_BASICAUTH ARG GRAFANA_PROM_BASICAUTH @@ -12,6 +9,12 @@ ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} +FROM ghcr.io/blockjoy/${RETH_IMAGE} AS reth-build + +FROM ghcr.io/blockjoy/${LIGHTHOUSE_IMAGE} AS lighthouse-build + +FROM ghcr.io/blockjoy/${BASE_IMAGE} + RUN mkdir -p /root/bin COPY --from=reth-build /root/bin/reth /root/bin/ COPY --from=lighthouse-build /root/bin/lighthouse /root/bin/ From 93e8135c866c9ea2402adaa1c4072d75cb5241ef Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 12:17:42 +0200 Subject: [PATCH 42/88] Fixed workflow --- .github/workflows/docker-build.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index ae8af734..08631e68 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -94,11 +94,11 @@ jobs: BUILD_CONTEXTS=$(grep -h "^FROM.*\${.*_IMAGE}" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do # Extract variable name from FROM line var_name=$(echo "$line" | grep -o '\${[^}]*}' | tr -d '${}\n') - # Extract image name from ARG statement + # Extract image name and remove any default tag image=$(grep "^ARG ${var_name}=" ./${{ matrix.image_path }}/Dockerfile | cut -d'=' -f2 | cut -d':' -f1) # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then - # Use short name for context key but specify it as a docker-image context + # Use short name for context key but specify it as a docker-image context with SHA tag echo "${image}=docker-image://ghcr.io/blockjoy/${image}:${SHA}" fi done | sort -u | tr '\n' ',' | sed 's/,$//') @@ -180,18 +180,18 @@ jobs: BUILD_CONTEXTS=$(grep -h "^FROM.*\${.*_IMAGE}" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do # Extract variable name from FROM line var_name=$(echo "$line" | grep -o '\${[^}]*}' | tr -d '${}\n') - # Extract image name from ARG statement + # Extract image name and remove any default tag image=$(grep "^ARG ${var_name}=" ./${{ matrix.image_path }}/Dockerfile | cut -d'=' -f2 | cut -d':' -f1) # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then - # Use short name for context key but specify it as a docker-image context + # Use short name for context key but specify it as a docker-image context with SHA tag echo "${image}=docker-image://ghcr.io/blockjoy/${image}:${SHA}" fi done | sort -u | tr '\n' ',' | sed 's/,$//') echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT echo "Generated build contexts: ${BUILD_CONTEXTS}" echo "Valid images: ${VALID_IMAGES}" - echo "Current image path: ${{ matrix.image_path }}" + echo "Current image path: ${{ matrix.image_path }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push client images @@ -252,18 +252,18 @@ jobs: BUILD_CONTEXTS=$(grep -h "^FROM.*\${.*_IMAGE}" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do # Extract variable name from FROM line var_name=$(echo "$line" | grep -o '\${[^}]*}' | tr -d '${}\n') - # Extract image name from ARG statement + # Extract image name and remove any default tag image=$(grep "^ARG ${var_name}=" ./${{ matrix.image_path }}/Dockerfile | cut -d'=' -f2 | cut -d':' -f1) # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then - # Use short name for context key but specify it as a docker-image context + # Use short name for context key but specify it as a docker-image context with SHA tag echo "${image}=docker-image://ghcr.io/blockjoy/${image}:${SHA}" fi done | sort -u | tr '\n' ',' | sed 's/,$//') echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT echo "Generated build contexts: ${BUILD_CONTEXTS}" echo "Valid images: ${VALID_IMAGES}" - echo "Current image path: ${{ matrix.image_path }}" + echo "Current image path: ${{ matrix.image_path }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push protocol images From 23d96dc39802d3c0aafd049e211b25d919281785 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 12:18:24 +0200 Subject: [PATCH 43/88] Fixed workflow --- .github/workflows/docker-build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 08631e68..ed8faf15 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -74,6 +74,7 @@ jobs: SHA=$(git rev-parse --short HEAD) echo "image_name=node-base" >> $GITHUB_OUTPUT echo "image_tag=${SHA}" >> $GITHUB_OUTPUT + echo "sha=${SHA}" >> $GITHUB_OUTPUT - name: Generate build contexts id: build-contexts run: | @@ -105,7 +106,7 @@ jobs: echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT echo "Generated build contexts: ${BUILD_CONTEXTS}" echo "Valid images: ${VALID_IMAGES}" - echo "Current image path: ${{ matrix.image_path }}" + echo "Current image path: ${{ matrix.image_path }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push base image @@ -160,6 +161,7 @@ jobs: fi echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT echo "image_tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT + echo "sha=${SHA}" >> $GITHUB_OUTPUT - name: Generate build contexts id: build-contexts run: | @@ -232,6 +234,7 @@ jobs: IMAGE_NAME=$(basename $(dirname ${{ matrix.image_path }}))"-"$(basename ${{ matrix.image_path }}) echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT echo "image_tag=${SHA}" >> $GITHUB_OUTPUT + echo "sha=${SHA}" >> $GITHUB_OUTPUT - name: Generate build contexts id: build-contexts run: | From 344510c96a5c57769a7a9266d800fb8d2f54e063 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 12:20:37 +0200 Subject: [PATCH 44/88] Fixed workflow --- .github/workflows/docker-build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index ed8faf15..dec94c46 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -106,7 +106,7 @@ jobs: echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT echo "Generated build contexts: ${BUILD_CONTEXTS}" echo "Valid images: ${VALID_IMAGES}" - echo "Current image path: ${{ matrix.image_path }} + echo "Current image path: '${{ matrix.image_path }}'" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push base image @@ -193,7 +193,7 @@ jobs: echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT echo "Generated build contexts: ${BUILD_CONTEXTS}" echo "Valid images: ${VALID_IMAGES}" - echo "Current image path: ${{ matrix.image_path }} + echo "Current image path: '${{ matrix.image_path }}'" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push client images @@ -266,7 +266,7 @@ jobs: echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT echo "Generated build contexts: ${BUILD_CONTEXTS}" echo "Valid images: ${VALID_IMAGES}" - echo "Current image path: ${{ matrix.image_path }} + echo "Current image path: '${{ matrix.image_path }}'" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push protocol images From 8bcc6420a107458e0550608c78db5156bbfcaf6d Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 12:22:56 +0200 Subject: [PATCH 45/88] Fixed workflow --- .github/workflows/docker-build.yml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index dec94c46..0679bd49 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -88,9 +88,6 @@ jobs: fi done | sort -u) - # Get the SHA for base images - SHA=$(git rev-parse --short HEAD) - # Now extract FROM directives and filter against our valid images BUILD_CONTEXTS=$(grep -h "^FROM.*\${.*_IMAGE}" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do # Extract variable name from FROM line @@ -100,7 +97,7 @@ jobs: # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then # Use short name for context key but specify it as a docker-image context with SHA tag - echo "${image}=docker-image://ghcr.io/blockjoy/${image}:${SHA}" + echo "${image}=docker-image://ghcr.io/blockjoy/${image}:${{ steps.version.outputs.sha }}" fi done | sort -u | tr '\n' ',' | sed 's/,$//') echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT @@ -175,9 +172,6 @@ jobs: fi done | sort -u) - # Get the SHA for base images - SHA=$(git rev-parse --short HEAD) - # Now extract FROM directives and filter against our valid images BUILD_CONTEXTS=$(grep -h "^FROM.*\${.*_IMAGE}" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do # Extract variable name from FROM line @@ -187,7 +181,7 @@ jobs: # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then # Use short name for context key but specify it as a docker-image context with SHA tag - echo "${image}=docker-image://ghcr.io/blockjoy/${image}:${SHA}" + echo "${image}=docker-image://ghcr.io/blockjoy/${image}:${{ steps.version.outputs.sha }}" fi done | sort -u | tr '\n' ',' | sed 's/,$//') echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT @@ -248,9 +242,6 @@ jobs: fi done | sort -u) - # Get the SHA for base images - SHA=$(git rev-parse --short HEAD) - # Now extract FROM directives and filter against our valid images BUILD_CONTEXTS=$(grep -h "^FROM.*\${.*_IMAGE}" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do # Extract variable name from FROM line @@ -260,7 +251,7 @@ jobs: # Check if this image is in our valid image list if echo "$VALID_IMAGES" | grep -q "^${image}$"; then # Use short name for context key but specify it as a docker-image context with SHA tag - echo "${image}=docker-image://ghcr.io/blockjoy/${image}:${SHA}" + echo "${image}=docker-image://ghcr.io/blockjoy/${image}:${{ steps.version.outputs.sha }}" fi done | sort -u | tr '\n' ',' | sed 's/,$//') echo "contexts=${BUILD_CONTEXTS}" >> $GITHUB_OUTPUT From abbc4aea58dd24a5cb473f5525aff02c70a3f081 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 12:25:10 +0200 Subject: [PATCH 46/88] Fixed workflow --- .github/workflows/docker-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 0679bd49..5d1dd948 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -199,6 +199,7 @@ jobs: GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }} GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }} CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }} + BASE_IMAGE=node-base:${{ steps.version.outputs.sha }} build-contexts: ${{ steps.build-contexts.outputs.contexts }} cache-from: type=gha cache-to: type=gha,mode=max From 96b77a09c236f01e742f64abf23679b9ef5c3270 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 13:00:35 +0200 Subject: [PATCH 47/88] Fixed protocol Dockerfiles --- .github/workflows/docker-build.yml | 8 +++++--- ethereum/ethereum-erigon/Dockerfile | 12 ++++++------ ethereum/ethereum-reth/Dockerfile | 12 ++++++------ 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 5d1dd948..38a1546b 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -141,10 +141,12 @@ jobs: run: | SHA=$(git rev-parse --short HEAD) CLIENT_NAME=$(basename ${{ matrix.image_path }}) - IMAGE_NAME="blockjoy-${CLIENT_NAME}" + IMAGE_NAME="${CLIENT_NAME}" - if [[ -f "${{ matrix.image_path }}/Dockerfile" ]]; then - CLIENT_VERSION=$(grep -E "ENV.*_VERSION=[[:space:]]*v?[0-9]+\.[0-9]+\.[0-9]+[-.a-zA-Z0-9]*" "${{ matrix.image_path }}/Dockerfile" | grep -oE "v?[0-9]+\.[0-9]+\.[0-9]+[-.a-zA-Z0-9]*") + if [[ "${{ matrix.image_path }}" =~ /exec/ ]]; then + # Only extract version for execution clients + CLIENT_UPPER=$(echo "${CLIENT_NAME}" | tr '[:lower:]' '[:upper:]') + CLIENT_VERSION=$(grep -E "ENV ${CLIENT_UPPER}_VERSION=[[:space:]]*v?[0-9]+\.[0-9]+\.[0-9]+[-.a-zA-Z0-9]*" "${{ matrix.image_path }}/Dockerfile" | grep -oE "v?[0-9]+\.[0-9]+\.[0-9]+[-.a-zA-Z0-9]*") if [[ ! -z "$CLIENT_VERSION" ]]; then # Remove 'v' prefix and ensure version is Docker-compatible (only alphanumeric and . - _) CLIENT_VERSION=${CLIENT_VERSION#v} diff --git a/ethereum/ethereum-erigon/Dockerfile b/ethereum/ethereum-erigon/Dockerfile index 703f7e1f..9b831da5 100644 --- a/ethereum/ethereum-erigon/Dockerfile +++ b/ethereum/ethereum-erigon/Dockerfile @@ -1,15 +1,15 @@ -ARG ERIGON_IMAGE=ethereum-erigon:latest -ARG LIGHTHOUSE_IMAGE=lighthouse:latest -ARG BASE_IMAGE=node-base:latest +ARG ERIGON_IMAGE=ghcr.io/blockjoy/erigon:latest +ARG LIGHTHOUSE_IMAGE=ghcr.io/blockjoy/lighthouse:latest +ARG BASE_IMAGE=ghcr.io/blockjoy/node-base:latest ARG CLOUDFLARE_API_KEY ARG GRAFANA_LOKI_BASICAUTH ARG GRAFANA_PROM_BASICAUTH -FROM ghcr.io/blockjoy/${ERIGON_IMAGE} as erigon-build +FROM ${ERIGON_IMAGE} as erigon-build -FROM ghcr.io/blockjoy/${LIGHTHOUSE_IMAGE} AS lighthouse-build +FROM ${LIGHTHOUSE_IMAGE} AS lighthouse-build -FROM ghcr.io/blockjoy/${BASE_IMAGE} +FROM ${BASE_IMAGE} ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} diff --git a/ethereum/ethereum-reth/Dockerfile b/ethereum/ethereum-reth/Dockerfile index 8bbec420..859c6db7 100644 --- a/ethereum/ethereum-reth/Dockerfile +++ b/ethereum/ethereum-reth/Dockerfile @@ -1,6 +1,6 @@ -ARG RETH_IMAGE=reth:latest -ARG LIGHTHOUSE_IMAGE=lighthouse:latest -ARG BASE_IMAGE=node-base:latest +ARG RETH_IMAGE=ghcr.io/blockjoy/reth:latest +ARG LIGHTHOUSE_IMAGE=ghcr.io/blockjoy/lighthouse:latest +ARG BASE_IMAGE=ghcr.io/blockjoy/node-base:latest ARG CLOUDFLARE_API_KEY ARG GRAFANA_LOKI_BASICAUTH ARG GRAFANA_PROM_BASICAUTH @@ -9,11 +9,11 @@ ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} -FROM ghcr.io/blockjoy/${RETH_IMAGE} AS reth-build +FROM ${RETH_IMAGE} AS reth-build -FROM ghcr.io/blockjoy/${LIGHTHOUSE_IMAGE} AS lighthouse-build +FROM ${LIGHTHOUSE_IMAGE} AS lighthouse-build -FROM ghcr.io/blockjoy/${BASE_IMAGE} +FROM ${BASE_IMAGE} RUN mkdir -p /root/bin COPY --from=reth-build /root/bin/reth /root/bin/ From 46c27ce7a2306bfcf631d56e5f9df6d4cca798f2 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 13:30:00 +0200 Subject: [PATCH 48/88] Run on self hosted --- .github/workflows/docker-build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 38a1546b..557ea8d1 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -20,7 +20,7 @@ concurrency: jobs: detect-changes: - runs-on: ubuntu-latest + runs-on: self-hosted outputs: base_matrix: ${{ steps.set-matrix.outputs.base_matrix }} clients_matrix: ${{ steps.set-matrix.outputs.clients_matrix }} @@ -57,7 +57,7 @@ jobs: build-base: needs: detect-changes if: ${{ fromJson(needs.detect-changes.outputs.base_matrix).include[0] }} - runs-on: ubuntu-latest + runs-on: self-hosted strategy: matrix: ${{ fromJson(needs.detect-changes.outputs.base_matrix) }} steps: @@ -125,7 +125,7 @@ jobs: always() && needs.build-base.result != 'failure' && fromJson(needs.detect-changes.outputs.clients_matrix).include[0] - runs-on: ubuntu-latest + runs-on: self-hosted strategy: matrix: ${{ fromJson(needs.detect-changes.outputs.clients_matrix) }} steps: @@ -213,7 +213,7 @@ jobs: always() && needs.build-clients.result != 'failure' && fromJson(needs.detect-changes.outputs.protocols_matrix).include[0] - runs-on: ubuntu-latest + runs-on: self-hosted strategy: matrix: ${{ fromJson(needs.detect-changes.outputs.protocols_matrix) }} steps: From 15bdc49cec0fc12b563f0faa8a50e39f4cac632e Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 13:35:45 +0200 Subject: [PATCH 49/88] Send URIs to protocol images URI --- .github/workflows/docker-build.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 557ea8d1..7bfcb706 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -261,6 +261,16 @@ jobs: echo "Generated build contexts: ${BUILD_CONTEXTS}" echo "Valid images: ${VALID_IMAGES}" echo "Current image path: '${{ matrix.image_path }}'" + - name: Generate build args + id: build-args + run: | + # Extract all *_IMAGE args from Dockerfile and set them with SHA tags + IMAGE_ARGS=$(grep "^ARG.*_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do + var_name=$(echo "$line" | cut -d'=' -f1 | cut -d' ' -f2) + image_name=$(echo "$line" | cut -d'=' -f2 | cut -d':' -f1 | xargs basename) + echo "${var_name}=ghcr.io/blockjoy/${image_name}:${{ steps.version.outputs.sha }}" + done | tr '\n' ',' | sed 's/,$//') + echo "image_args=${IMAGE_ARGS}" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push protocol images @@ -272,6 +282,7 @@ jobs: GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }} GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }} CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }} + ${{ steps.build-args.outputs.image_args }} build-contexts: ${{ steps.build-contexts.outputs.contexts }} cache-from: type=gha cache-to: type=gha,mode=max From 75e638d1da0ffbde9c1b5f2196054ed5ced4a566 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 14:59:50 +0200 Subject: [PATCH 50/88] Improved builds, fixes for protocol dockerfiles --- .github/workflows/docker-build.yml | 28 +++++++++++++++++++--------- clients/exec/erigon/Dockerfile | 27 +++++++++++---------------- clients/exec/reth/Dockerfile | 29 ++++++++++++++++++----------- 3 files changed, 48 insertions(+), 36 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 7bfcb706..57303939 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -7,6 +7,9 @@ on: - '**.md' - '.github/**' - '!.github/workflows/docker-build.yml' + - 'docs/**' + - '.gitignore' + - 'LICENSE' permissions: contents: read @@ -33,19 +36,24 @@ jobs: - name: Get changed files id: changed-files uses: tj-actions/changed-files@v40 - with: - files: | - ** - name: Generate build matrices id: set-matrix run: | - # Get list of changed files and filter to unique directories containing Dockerfile + # First find all directories containing Dockerfiles + DOCKER_DIRS=$(find . -name Dockerfile -exec dirname {} \;) + + # Get list of changed files and map them to Dockerfile directories CHANGED_DIRS=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | while read -r file; do dir=$(dirname "$file") - if [[ -f "$dir/Dockerfile" ]]; then - echo "$dir" - fi + # Walk up the directory tree until we find a Dockerfile or reach root + while [[ "$dir" != "." && "$dir" != "/" ]]; do + if echo "$DOCKER_DIRS" | grep -q "^${dir}$"; then + echo "$dir" + break + fi + dir=$(dirname "$dir") + done done | sort -u) # Create matrices using jq, filtering by directory structure @@ -269,8 +277,10 @@ jobs: var_name=$(echo "$line" | cut -d'=' -f1 | cut -d' ' -f2) image_name=$(echo "$line" | cut -d'=' -f2 | cut -d':' -f1 | xargs basename) echo "${var_name}=ghcr.io/blockjoy/${image_name}:${{ steps.version.outputs.sha }}" - done | tr '\n' ',' | sed 's/,$//') - echo "image_args=${IMAGE_ARGS}" >> $GITHUB_OUTPUT + done | tr '\n' '\n') + echo "image_args<> $GITHUB_OUTPUT + echo "$IMAGE_ARGS" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push protocol images diff --git a/clients/exec/erigon/Dockerfile b/clients/exec/erigon/Dockerfile index 8538ae07..058f69b6 100644 --- a/clients/exec/erigon/Dockerfile +++ b/clients/exec/erigon/Dockerfile @@ -1,29 +1,24 @@ ARG BASE_IMAGE=node-base:latest -FROM ghcr.io/blockjoy/${BASE_IMAGE} - -# Add build arguments for Cloudflare and Grafana push credentials -ARG CLOUDFLARE_API_KEY -ARG GRAFANA_LOKI_BASICAUTH -ARG GRAFANA_PROM_BASICAUTH +FROM ghcr.io/blockjoy/${BASE_IMAGE} as builder ENV DEBIAN_FRONTEND=noninteractive -ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} -ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} -ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} - ENV ERIGON_VERSION=v2.60.9 - ENV PATH="/usr/local/go/bin:${PATH}" ENV GOROOT="/usr/local/go" RUN mkdir -p /root/bin +# Clone and checkout in a separate layer WORKDIR /root -RUN git clone https://github.com/erigontech/erigon.git && \ - cd erigon && \ - git checkout $ERIGON_VERSION +RUN git clone --depth 1 --branch $ERIGON_VERSION https://github.com/erigontech/erigon.git +# Build erigon with Go caching WORKDIR /root/erigon -RUN make erigon +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + make erigon && \ + cp build/bin/erigon /root/bin/ -RUN cp /root/erigon/build/bin/erigon /root/bin/erigon +# Create minimal final image +FROM ghcr.io/blockjoy/${BASE_IMAGE} +COPY --from=builder /root/bin/erigon /root/bin/erigon diff --git a/clients/exec/reth/Dockerfile b/clients/exec/reth/Dockerfile index 58f214f4..5c20eda5 100644 --- a/clients/exec/reth/Dockerfile +++ b/clients/exec/reth/Dockerfile @@ -1,32 +1,39 @@ ARG BASE_IMAGE=node-base:latest -FROM ghcr.io/blockjoy/${BASE_IMAGE} +FROM ghcr.io/blockjoy/${BASE_IMAGE} as builder ENV DEBIAN_FRONTEND=noninteractive - ENV RETH_VERSION=v1.1.4 -ENV LIGHTHOUSE_VERSION=v5.3.0 ENV CARGO_NET_GIT_FETCH_WITH_CLI=true -# Install system dependencies including libclang +# Install system dependencies in a separate layer RUN apt-get update && \ apt-get install -y \ clang \ libclang-dev \ && rm -rf /var/lib/apt/lists/* -# Install Rust and required dependencies +# Install Rust in a separate layer RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \ . "$HOME/.cargo/env" && \ rustup default stable && \ rustup target add x86_64-unknown-linux-gnu +# Clone and checkout in a separate layer WORKDIR /root -RUN git clone https://github.com/paradigmxyz/reth.git && \ - cd reth && \ - git checkout $RETH_VERSION +RUN --mount=type=cache,target=/root/.cargo/git \ + --mount=type=cache,target=/root/.cargo/registry \ + git clone --depth 1 --branch $RETH_VERSION https://github.com/paradigmxyz/reth.git -# Build reth with performance features +# Build reth with cargo caching WORKDIR /root/reth -RUN . "$HOME/.cargo/env" && cargo build --profile maxperf && \ +RUN --mount=type=cache,target=/root/.cargo/git \ + --mount=type=cache,target=/root/.cargo/registry \ + --mount=type=cache,target=/root/reth/target \ + . "$HOME/.cargo/env" && \ + cargo build --profile maxperf && \ mkdir -p /root/bin && \ - cp /root/reth/target/maxperf/reth /root/bin/reth + cp target/maxperf/reth /root/bin/ + +# Create minimal final image +FROM ghcr.io/blockjoy/${BASE_IMAGE} +COPY --from=builder /root/bin/reth /root/bin/reth From 28c8f2df0cfa52882af57a8b28f6f33b1edf6d05 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 15:02:12 +0200 Subject: [PATCH 51/88] Fix matrix empty path --- .github/workflows/docker-build.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 57303939..9b538438 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -236,8 +236,14 @@ jobs: id: version run: | SHA=$(git rev-parse --short HEAD) - IMAGE_NAME=$(basename $(dirname ${{ matrix.image_path }}))"-"$(basename ${{ matrix.image_path }}) - echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT + if [[ -n "${{ matrix.image_path }}" ]]; then + # Only try to generate image name if matrix path is not empty + IMAGE_NAME=$(basename $(dirname "${{ matrix.image_path }}"))"-"$(basename "${{ matrix.image_path }}") + echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT + else + echo "::error ::No image path provided in matrix" + exit 1 + fi echo "image_tag=${SHA}" >> $GITHUB_OUTPUT echo "sha=${SHA}" >> $GITHUB_OUTPUT - name: Generate build contexts From a50335c5783d19f4a3d9e85f15705da285e51d43 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 15:05:09 +0200 Subject: [PATCH 52/88] Improved matrix of changed protocols logic --- .github/workflows/docker-build.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 9b538438..b4c6e9c0 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -57,10 +57,18 @@ jobs: done | sort -u) # Create matrices using jq, filtering by directory structure - echo "base_matrix=$(echo "$CHANGED_DIRS" | grep "^node-base$" | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT - echo "clients_matrix=$(echo "$CHANGED_DIRS" | grep "^clients/" | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT + echo "base_matrix=$(echo "$CHANGED_DIRS" | grep "^./node-base$" | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT + echo "clients_matrix=$(echo "$CHANGED_DIRS" | grep "^./clients/" | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT + # Any directory that's not node-base or clients/ and contains a Dockerfile is a protocol - echo "protocols_matrix=$(echo "$CHANGED_DIRS" | grep -v "^node-base$" | grep -v "^clients/" | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT + # Make sure to exclude empty results by checking if there are any matches + PROTOCOLS=$(echo "$CHANGED_DIRS" | grep -v "^./node-base$" | grep -v "^./clients/") + if [[ -n "$PROTOCOLS" ]]; then + echo "protocols_matrix=$(echo "$PROTOCOLS" | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT + else + # Provide empty but valid matrix when no protocols changed + echo 'protocols_matrix={"include":[]}' >> $GITHUB_OUTPUT + fi build-base: needs: detect-changes From becd6871d5f28185967f502583266334f83d8b3c Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 15:06:46 +0200 Subject: [PATCH 53/88] Dummy change for testing --- ethereum/ethereum-reth/reth.rhai | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index b199716c..1a8191b4 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy", + name: "alloy-test", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ @@ -36,4 +36,4 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { } ] } -} \ No newline at end of file +} From dd8ddf4656b042a5466ff50f41d64f9d014f65e1 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 15:12:52 +0200 Subject: [PATCH 54/88] Docs --- .github/workflows/docker-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index b4c6e9c0..a2257ca5 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -40,10 +40,10 @@ jobs: - name: Generate build matrices id: set-matrix run: | - # First find all directories containing Dockerfiles + # First find all Docker build contexts (directories containing Dockerfile) DOCKER_DIRS=$(find . -name Dockerfile -exec dirname {} \;) - # Get list of changed files and map them to Dockerfile directories + # Get list of changed files and map them to build contexts CHANGED_DIRS=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | while read -r file; do dir=$(dirname "$file") # Walk up the directory tree until we find a Dockerfile or reach root From 8ddb6fbe1c87cd304ec33922938f3e016e01ede6 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 15:13:47 +0200 Subject: [PATCH 55/88] Dummy change for testing --- ethereum/ethereum-reth/reth.rhai | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index 1a8191b4..d47ebfca 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-test", + name: "alloy", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From f671280e257542909adc6d1f412414d4976698bd Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 15:17:46 +0200 Subject: [PATCH 56/88] Improved protocol matrix, test --- ethereum/ethereum-reth/reth.rhai | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index d47ebfca..aa8115bf 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy", + name: "alloy-t1", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From 4d30b3be359978ec7ead4a0c29df54f5a3b14343 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 15:19:26 +0200 Subject: [PATCH 57/88] Fixed workflow --- .github/workflows/docker-build.yml | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index a2257ca5..48bd4e8d 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -41,7 +41,9 @@ jobs: id: set-matrix run: | # First find all Docker build contexts (directories containing Dockerfile) - DOCKER_DIRS=$(find . -name Dockerfile -exec dirname {} \;) + DOCKER_DIRS=$(find . -name Dockerfile -exec dirname {} \; | sort) + echo "Found Docker directories:" + echo "$DOCKER_DIRS" # Get list of changed files and map them to build contexts CHANGED_DIRS=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | while read -r file; do @@ -55,18 +57,24 @@ jobs: dir=$(dirname "$dir") done done | sort -u) + echo "Changed directories:" + echo "$CHANGED_DIRS" # Create matrices using jq, filtering by directory structure - echo "base_matrix=$(echo "$CHANGED_DIRS" | grep "^./node-base$" | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT - echo "clients_matrix=$(echo "$CHANGED_DIRS" | grep "^./clients/" | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT + BASE=$(echo "$CHANGED_DIRS" | grep "^./node-base$" || true) + CLIENTS=$(echo "$CHANGED_DIRS" | grep "^./clients/" || true) + PROTOCOLS=$(echo "$CHANGED_DIRS" | grep "^./[^/]\+/[^/]\+-[^/]\+$" || true) + + echo "Base dirs: $BASE" + echo "Client dirs: $CLIENTS" + echo "Protocol dirs: $PROTOCOLS" + + echo "base_matrix=$(echo "$BASE" | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT + echo "clients_matrix=$(echo "$CLIENTS" | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT - # Any directory that's not node-base or clients/ and contains a Dockerfile is a protocol - # Make sure to exclude empty results by checking if there are any matches - PROTOCOLS=$(echo "$CHANGED_DIRS" | grep -v "^./node-base$" | grep -v "^./clients/") if [[ -n "$PROTOCOLS" ]]; then echo "protocols_matrix=$(echo "$PROTOCOLS" | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT else - # Provide empty but valid matrix when no protocols changed echo 'protocols_matrix={"include":[]}' >> $GITHUB_OUTPUT fi From cbbec97fdbce90ddc52a64081742a4810cb4c302 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 15:19:42 +0200 Subject: [PATCH 58/88] Test --- ethereum/ethereum-reth/reth.rhai | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index aa8115bf..312e663c 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t1", + name: "alloy-t2", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From 5c7fc4ea5651f0f8128cbb5ba0495a50179286ea Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 15:25:19 +0200 Subject: [PATCH 59/88] Increased workflow verbosity --- .github/workflows/docker-build.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 48bd4e8d..40cc0bc7 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -45,12 +45,19 @@ jobs: echo "Found Docker directories:" echo "$DOCKER_DIRS" + echo "Changed files from action:" + echo "${{ steps.changed-files.outputs.all_changed_files }}" + # Get list of changed files and map them to build contexts CHANGED_DIRS=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | while read -r file; do dir=$(dirname "$file") + echo "Processing file: $file" + echo "Initial dir: $dir" # Walk up the directory tree until we find a Dockerfile or reach root while [[ "$dir" != "." && "$dir" != "/" ]]; do + echo "Checking dir: $dir" if echo "$DOCKER_DIRS" | grep -q "^${dir}$"; then + echo "Found Docker dir: $dir" echo "$dir" break fi From 0d5196f24fabc5bed5bcbec7296c2e542afc8657 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 15:28:22 +0200 Subject: [PATCH 60/88] Fixed matrices --- .github/workflows/docker-build.yml | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 40cc0bc7..763a5cf6 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -56,11 +56,13 @@ jobs: # Walk up the directory tree until we find a Dockerfile or reach root while [[ "$dir" != "." && "$dir" != "/" ]]; do echo "Checking dir: $dir" - if echo "$DOCKER_DIRS" | grep -q "^${dir}$"; then - echo "Found Docker dir: $dir" - echo "$dir" - break - fi + echo "$DOCKER_DIRS" | while read -r docker_dir; do + if [[ "$dir" == "$docker_dir" ]]; then + echo "Found Docker dir: $dir" + echo "$dir" + break 2 + fi + done dir=$(dirname "$dir") done done | sort -u) @@ -76,8 +78,18 @@ jobs: echo "Client dirs: $CLIENTS" echo "Protocol dirs: $PROTOCOLS" - echo "base_matrix=$(echo "$BASE" | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT - echo "clients_matrix=$(echo "$CLIENTS" | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT + # Only output matrices if we have changes + if [[ -n "$BASE" ]]; then + echo "base_matrix=$(echo "$BASE" | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT + else + echo 'base_matrix={"include":[]}' >> $GITHUB_OUTPUT + fi + + if [[ -n "$CLIENTS" ]]; then + echo "clients_matrix=$(echo "$CLIENTS" | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT + else + echo 'clients_matrix={"include":[]}' >> $GITHUB_OUTPUT + fi if [[ -n "$PROTOCOLS" ]]; then echo "protocols_matrix=$(echo "$PROTOCOLS" | jq -Rsc 'split("\n")[:-1] | {include: map({image_path: .})}')" >> $GITHUB_OUTPUT From ce9e36202c2f9c11bf5ec0f7bbe4101bdcbdd600 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 15:29:29 +0200 Subject: [PATCH 61/88] Dummy change for testing --- ethereum/ethereum-reth/reth.rhai | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index 312e663c..ca1fbc57 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t2", + name: "alloy-t3", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From 53670bd32dbd608656058edcf3141e408c50b71e Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 15:32:02 +0200 Subject: [PATCH 62/88] Fixed dir detection --- .github/workflows/docker-build.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 763a5cf6..5f9afc84 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -56,13 +56,18 @@ jobs: # Walk up the directory tree until we find a Dockerfile or reach root while [[ "$dir" != "." && "$dir" != "/" ]]; do echo "Checking dir: $dir" - echo "$DOCKER_DIRS" | while read -r docker_dir; do + found=false + while IFS= read -r docker_dir; do if [[ "$dir" == "$docker_dir" ]]; then echo "Found Docker dir: $dir" echo "$dir" - break 2 + found=true + break fi - done + done <<< "$DOCKER_DIRS" + if [[ "$found" == "true" ]]; then + break + fi dir=$(dirname "$dir") done done | sort -u) From 3c7bfd4476632c9997e237c8c146cd31f7143f7e Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 15:32:12 +0200 Subject: [PATCH 63/88] Dummy change for testing --- ethereum/ethereum-reth/reth.rhai | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index ca1fbc57..fde55dc2 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t3", + name: "alloy-t4", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From f38494b12785d9d7b1cde3be34fff17ea6d6bf52 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 15:36:41 +0200 Subject: [PATCH 64/88] Simplified changed file detection, test commit --- .github/workflows/docker-build.yml | 24 +++++------------------- ethereum/ethereum-reth/reth.rhai | 2 +- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 5f9afc84..ee2f4076 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -45,30 +45,16 @@ jobs: echo "Found Docker directories:" echo "$DOCKER_DIRS" + # Get list of changed files echo "Changed files from action:" echo "${{ steps.changed-files.outputs.all_changed_files }}" - # Get list of changed files and map them to build contexts + # For each changed file, find which Docker directory it belongs to CHANGED_DIRS=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | while read -r file; do - dir=$(dirname "$file") - echo "Processing file: $file" - echo "Initial dir: $dir" - # Walk up the directory tree until we find a Dockerfile or reach root - while [[ "$dir" != "." && "$dir" != "/" ]]; do - echo "Checking dir: $dir" - found=false - while IFS= read -r docker_dir; do - if [[ "$dir" == "$docker_dir" ]]; then - echo "Found Docker dir: $dir" - echo "$dir" - found=true - break - fi - done <<< "$DOCKER_DIRS" - if [[ "$found" == "true" ]]; then - break + echo "$DOCKER_DIRS" | while read -r docker_dir; do + if [[ "$file" == "$docker_dir"/* || "$file" == "$docker_dir" ]]; then + echo "$docker_dir" fi - dir=$(dirname "$dir") done done | sort -u) echo "Changed directories:" diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index fde55dc2..f0345e19 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t4", + name: "alloy-t5", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From 635ae280e2134cac8a12444647b493c141a9b2ef Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 15:39:53 +0200 Subject: [PATCH 65/88] Improved path matching --- .github/workflows/docker-build.yml | 13 ++++++++----- ethereum/ethereum-reth/reth.rhai | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index ee2f4076..edc63bd5 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -41,7 +41,7 @@ jobs: id: set-matrix run: | # First find all Docker build contexts (directories containing Dockerfile) - DOCKER_DIRS=$(find . -name Dockerfile -exec dirname {} \; | sort) + DOCKER_DIRS=$(find . -name Dockerfile -exec dirname {} \; | sed 's|^./||' | sort) echo "Found Docker directories:" echo "$DOCKER_DIRS" @@ -51,8 +51,11 @@ jobs: # For each changed file, find which Docker directory it belongs to CHANGED_DIRS=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | while read -r file; do + echo "Checking file: $file" echo "$DOCKER_DIRS" | while read -r docker_dir; do - if [[ "$file" == "$docker_dir"/* || "$file" == "$docker_dir" ]]; then + echo "Against dir: $docker_dir" + if [[ "$file" =~ ^"$docker_dir"(/|$) ]]; then + echo "Match found: $docker_dir" echo "$docker_dir" fi done @@ -61,9 +64,9 @@ jobs: echo "$CHANGED_DIRS" # Create matrices using jq, filtering by directory structure - BASE=$(echo "$CHANGED_DIRS" | grep "^./node-base$" || true) - CLIENTS=$(echo "$CHANGED_DIRS" | grep "^./clients/" || true) - PROTOCOLS=$(echo "$CHANGED_DIRS" | grep "^./[^/]\+/[^/]\+-[^/]\+$" || true) + BASE=$(echo "$CHANGED_DIRS" | grep "^node-base$" || true) + CLIENTS=$(echo "$CHANGED_DIRS" | grep "^clients/" || true) + PROTOCOLS=$(echo "$CHANGED_DIRS" | grep "^[^/]\+/[^/]\+-[^/]\+$" || true) echo "Base dirs: $BASE" echo "Client dirs: $CLIENTS" diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index f0345e19..de5e1402 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t5", + name: "alloy-t6", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From 84b4a286ee44ac1607f24f95b175e82515d1c5fb Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 15:42:35 +0200 Subject: [PATCH 66/88] Improved changed file detection --- .github/workflows/docker-build.yml | 13 +++++++------ ethereum/ethereum-reth/reth.rhai | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index edc63bd5..9fbf2df4 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -36,7 +36,10 @@ jobs: - name: Get changed files id: changed-files uses: tj-actions/changed-files@v40 - + with: + files: | + **/* + - name: Generate build matrices id: set-matrix run: | @@ -46,16 +49,14 @@ jobs: echo "$DOCKER_DIRS" # Get list of changed files - echo "Changed files from action:" - echo "${{ steps.changed-files.outputs.all_changed_files }}" + echo "Changed files in this commit:" + echo "${{ steps.changed-files.outputs.all_changed_and_modified_files }}" # For each changed file, find which Docker directory it belongs to - CHANGED_DIRS=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | while read -r file; do + CHANGED_DIRS=$(echo "${{ steps.changed-files.outputs.all_changed_and_modified_files }}" | tr ' ' '\n' | while read -r file; do echo "Checking file: $file" echo "$DOCKER_DIRS" | while read -r docker_dir; do - echo "Against dir: $docker_dir" if [[ "$file" =~ ^"$docker_dir"(/|$) ]]; then - echo "Match found: $docker_dir" echo "$docker_dir" fi done diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index de5e1402..4cdc94b6 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t6", + name: "alloy-t7", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From 722585d06d30086841138031784cd24b5d0812e5 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 15:55:45 +0200 Subject: [PATCH 67/88] Improvements, test --- .github/workflows/docker-build.yml | 10 ++-------- ethereum/ethereum-reth/reth.rhai | 2 +- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 9fbf2df4..647d91e1 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -159,10 +159,7 @@ jobs: build-clients: needs: [detect-changes, build-base] - if: | - always() && - needs.build-base.result != 'failure' && - fromJson(needs.detect-changes.outputs.clients_matrix).include[0] + if: needs.build-base.result != 'failure' && fromJson(needs.detect-changes.outputs.clients_matrix).include[0] runs-on: self-hosted strategy: matrix: ${{ fromJson(needs.detect-changes.outputs.clients_matrix) }} @@ -247,10 +244,7 @@ jobs: build-protocols: needs: [detect-changes, build-clients] - if: | - always() && - needs.build-clients.result != 'failure' && - fromJson(needs.detect-changes.outputs.protocols_matrix).include[0] + if: needs.build-clients.result != 'failure' && fromJson(needs.detect-changes.outputs.protocols_matrix).include[0] runs-on: self-hosted strategy: matrix: ${{ fromJson(needs.detect-changes.outputs.protocols_matrix) }} diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index 4cdc94b6..a95fd999 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t7", + name: "alloy-t8", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From df60e71957515f35c8c1fa17ee69445354047894 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 15:59:58 +0200 Subject: [PATCH 68/88] Dummy change for testing --- ethereum/ethereum-reth/reth.rhai | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index a95fd999..6820cc9d 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t8", + name: "alloy-t9", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From 4b5f9f551d44130f758d3612198ee7711e681cab Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 16:03:30 +0200 Subject: [PATCH 69/88] Improved file detections --- .github/workflows/docker-build.yml | 9 ++++++--- ethereum/ethereum-reth/reth.rhai | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 647d91e1..ab2fea2a 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -39,7 +39,9 @@ jobs: with: files: | **/* - + json: true + dir_names: false + - name: Generate build matrices id: set-matrix run: | @@ -50,10 +52,11 @@ jobs: # Get list of changed files echo "Changed files in this commit:" - echo "${{ steps.changed-files.outputs.all_changed_and_modified_files }}" + CHANGED_FILES=$(echo '${{ steps.changed-files.outputs.modified_files }}' | jq -r '.[]' 2>/dev/null || echo '') + echo "$CHANGED_FILES" # For each changed file, find which Docker directory it belongs to - CHANGED_DIRS=$(echo "${{ steps.changed-files.outputs.all_changed_and_modified_files }}" | tr ' ' '\n' | while read -r file; do + CHANGED_DIRS=$(echo "$CHANGED_FILES" | while read -r file; do echo "Checking file: $file" echo "$DOCKER_DIRS" | while read -r docker_dir; do if [[ "$file" =~ ^"$docker_dir"(/|$) ]]; then diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index 6820cc9d..cc0ec714 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t9", + name: "alloy-t10", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From dfad82498494f4c5c77cf9e01411dea6d8bf4f46 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 16:05:40 +0200 Subject: [PATCH 70/88] Improved file detections --- .github/workflows/docker-build.yml | 6 +++++- ethereum/ethereum-reth/reth.rhai | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index ab2fea2a..0adba3c4 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -52,7 +52,11 @@ jobs: # Get list of changed files echo "Changed files in this commit:" - CHANGED_FILES=$(echo '${{ steps.changed-files.outputs.modified_files }}' | jq -r '.[]' 2>/dev/null || echo '') + CHANGED_FILES=$(echo '${{ steps.changed-files.outputs.all_modified_files }}' | jq -r '.[]' 2>/dev/null || echo '') + if [ -z "$CHANGED_FILES" ]; then + echo "Falling back to space-separated list" + CHANGED_FILES="${{ steps.changed-files.outputs.all_modified_files }}" + fi echo "$CHANGED_FILES" # For each changed file, find which Docker directory it belongs to diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index cc0ec714..7390956a 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t10", + name: "alloy-t11", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From 702365fd8f87d5eb81e8009438561a95eeea031b Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 16:08:09 +0200 Subject: [PATCH 71/88] Debugging --- .github/workflows/docker-build.yml | 16 ++++++++++++++++ ethereum/ethereum-reth/reth.rhai | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 0adba3c4..8bb7d6d6 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -42,6 +42,22 @@ jobs: json: true dir_names: false + - name: Debug changed files outputs + run: | + echo "Debug outputs from changed-files action:" + echo "all_modified_files:" + echo "${{ steps.changed-files.outputs.all_modified_files }}" + echo "modified_files:" + echo "${{ steps.changed-files.outputs.modified_files }}" + echo "added_files:" + echo "${{ steps.changed-files.outputs.added_files }}" + echo "deleted_files:" + echo "${{ steps.changed-files.outputs.deleted_files }}" + echo "renamed_files:" + echo "${{ steps.changed-files.outputs.renamed_files }}" + echo "copied_files:" + echo "${{ steps.changed-files.outputs.copied_files }}" + - name: Generate build matrices id: set-matrix run: | diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index 7390956a..d42e8e9c 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t11", + name: "alloy-t12", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From 490ba72acdc77d6a5fa8ede0d6ea85844d1eaca7 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 16:11:40 +0200 Subject: [PATCH 72/88] Fixed workflow --- .github/workflows/docker-build.yml | 3 +++ ethereum/ethereum-reth/reth.rhai | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 8bb7d6d6..d64f6d7b 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -41,6 +41,9 @@ jobs: **/* json: true dir_names: false + since_last_remote_commit: true + base_sha: ${{ github.event.pull_request.base.sha || github.event.before }} + sha: ${{ github.event.pull_request.head.sha || github.sha }} - name: Debug changed files outputs run: | diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index d42e8e9c..015bbf0b 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t12", + name: "alloy-t13", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From a9e7ca98c3c2eec054c6cf9f66323a106dc51443 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 16:14:17 +0200 Subject: [PATCH 73/88] Fixed workflow --- .github/workflows/docker-build.yml | 5 +---- ethereum/ethereum-reth/reth.rhai | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index d64f6d7b..f5bafc09 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -31,7 +31,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 0 + fetch-depth: 2 - name: Get changed files id: changed-files @@ -41,9 +41,6 @@ jobs: **/* json: true dir_names: false - since_last_remote_commit: true - base_sha: ${{ github.event.pull_request.base.sha || github.event.before }} - sha: ${{ github.event.pull_request.head.sha || github.sha }} - name: Debug changed files outputs run: | diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index 015bbf0b..9ff5d0d9 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t13", + name: "alloy-t14", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From c90b8eb6d1238f891273cb0affaa90738b51b74a Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 16:16:01 +0200 Subject: [PATCH 74/88] Fixed workflow --- .github/workflows/docker-build.yml | 2 ++ ethereum/ethereum-reth/reth.rhai | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index f5bafc09..01d39e4e 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -41,6 +41,8 @@ jobs: **/* json: true dir_names: false + base_sha: ${{ github.event.pull_request.base.sha }} + sha: ${{ github.event.pull_request.head.sha }} - name: Debug changed files outputs run: | diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index 9ff5d0d9..043648ff 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t14", + name: "alloy-t15", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From a5b9ce077b061fca9defbccb091d11c36acd75ba Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 16:23:50 +0200 Subject: [PATCH 75/88] Fixed workflow --- .github/workflows/docker-build.yml | 1 + ethereum/ethereum-reth/reth.rhai | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 01d39e4e..c353f2ae 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -43,6 +43,7 @@ jobs: dir_names: false base_sha: ${{ github.event.pull_request.base.sha }} sha: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 - name: Debug changed files outputs run: | diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index 043648ff..7fb5c2fd 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t15", + name: "alloy-t16", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From c083f6e0c6415dfe310f9faf4d0be12fd12ce65f Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 16:25:58 +0200 Subject: [PATCH 76/88] Fixed workflow --- .github/workflows/docker-build.yml | 2 -- ethereum/ethereum-reth/reth.rhai | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index c353f2ae..2f6fcd18 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -41,8 +41,6 @@ jobs: **/* json: true dir_names: false - base_sha: ${{ github.event.pull_request.base.sha }} - sha: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 - name: Debug changed files outputs diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index 7fb5c2fd..9a06aeb3 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t16", + name: "alloy-t17", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From fba1f7fed7a574927b7392636ba27e6fe21a580d Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 16:27:19 +0200 Subject: [PATCH 77/88] Fixed workflow --- .github/workflows/docker-build.yml | 2 +- ethereum/ethereum-reth/reth.rhai | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 2f6fcd18..ca93abe3 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -41,7 +41,7 @@ jobs: **/* json: true dir_names: false - fetch-depth: 0 + fetch-depth: 1 - name: Debug changed files outputs run: | diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index 9a06aeb3..5a11e7d0 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t17", + name: "alloy-t18", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From 8325e4cb41ca123fb0a9de52800328609d03cbed Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 16:29:05 +0200 Subject: [PATCH 78/88] Fixed workflow --- .github/workflows/docker-build.yml | 2 +- ethereum/ethereum-reth/reth.rhai | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index ca93abe3..c2f7be01 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -41,7 +41,7 @@ jobs: **/* json: true dir_names: false - fetch-depth: 1 + fetch_depth: 1 - name: Debug changed files outputs run: | diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index 5a11e7d0..6dd22ede 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t18", + name: "alloy-t19", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From a9ac8f35e073813833bf4fc3cbd4fd38d5329e17 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 16:29:40 +0200 Subject: [PATCH 79/88] Fixed workflow --- .github/workflows/docker-build.yml | 2 +- ethereum/ethereum-reth/reth.rhai | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index c2f7be01..d00a817f 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -41,7 +41,7 @@ jobs: **/* json: true dir_names: false - fetch_depth: 1 + fetch_depth: 0 - name: Debug changed files outputs run: | diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index 6dd22ede..81afe657 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t19", + name: "alloy-t20", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From 2738056e9d01bbde8cc988574996c6214fd8632d Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 16:31:47 +0200 Subject: [PATCH 80/88] Fixed workflow --- .github/workflows/docker-build.yml | 5 +++-- ethereum/ethereum-reth/reth.rhai | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index d00a817f..3626bd0d 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -29,9 +29,10 @@ jobs: clients_matrix: ${{ steps.set-matrix.outputs.clients_matrix }} protocols_matrix: ${{ steps.set-matrix.outputs.protocols_matrix }} steps: - - uses: actions/checkout@v4 + - name: Checkout + uses: actions/checkout@v4 with: - fetch-depth: 2 + fetch-depth: 0 - name: Get changed files id: changed-files diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index 81afe657..eac0a91d 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t20", + name: "alloy-t21", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From 6019a9fd36b94146103c95803e28b4cf7377055e Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 16:40:25 +0200 Subject: [PATCH 81/88] Fixed workflow --- .github/workflows/docker-build.yml | 46 +++++++++++------------------- ethereum/ethereum-reth/reth.rhai | 2 +- 2 files changed, 18 insertions(+), 30 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 3626bd0d..6e2c0607 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -31,37 +31,30 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Get changed files id: changed-files - uses: tj-actions/changed-files@v40 - with: - files: | - **/* - json: true - dir_names: false - fetch_depth: 0 + uses: tj-actions/changed-files@v45 + + - name: List all changed files + env: + ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + run: | + for file in ${ALL_CHANGED_FILES}; do + echo "$file was changed" + done - name: Debug changed files outputs + env: + ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} run: | echo "Debug outputs from changed-files action:" - echo "all_modified_files:" - echo "${{ steps.changed-files.outputs.all_modified_files }}" - echo "modified_files:" - echo "${{ steps.changed-files.outputs.modified_files }}" - echo "added_files:" - echo "${{ steps.changed-files.outputs.added_files }}" - echo "deleted_files:" - echo "${{ steps.changed-files.outputs.deleted_files }}" - echo "renamed_files:" - echo "${{ steps.changed-files.outputs.renamed_files }}" - echo "copied_files:" - echo "${{ steps.changed-files.outputs.copied_files }}" + echo "Changed files:" + echo "$ALL_CHANGED_FILES" - name: Generate build matrices id: set-matrix + env: + ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} run: | # First find all Docker build contexts (directories containing Dockerfile) DOCKER_DIRS=$(find . -name Dockerfile -exec dirname {} \; | sed 's|^./||' | sort) @@ -70,15 +63,10 @@ jobs: # Get list of changed files echo "Changed files in this commit:" - CHANGED_FILES=$(echo '${{ steps.changed-files.outputs.all_modified_files }}' | jq -r '.[]' 2>/dev/null || echo '') - if [ -z "$CHANGED_FILES" ]; then - echo "Falling back to space-separated list" - CHANGED_FILES="${{ steps.changed-files.outputs.all_modified_files }}" - fi - echo "$CHANGED_FILES" + echo "$ALL_CHANGED_FILES" # For each changed file, find which Docker directory it belongs to - CHANGED_DIRS=$(echo "$CHANGED_FILES" | while read -r file; do + CHANGED_DIRS=$(for file in ${ALL_CHANGED_FILES}; do echo "Checking file: $file" echo "$DOCKER_DIRS" | while read -r docker_dir; do if [[ "$file" =~ ^"$docker_dir"(/|$) ]]; then diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index eac0a91d..6fe23699 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t21", + name: "alloy-t22", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From 5805fc815707d2674f40a78b2753860b9bc7a0ae Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 16:44:01 +0200 Subject: [PATCH 82/88] Fixed workflow --- .github/workflows/docker-build.yml | 2 ++ ethereum/ethereum-reth/reth.rhai | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 6e2c0607..8ca5021f 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -34,6 +34,8 @@ jobs: - name: Get changed files id: changed-files uses: tj-actions/changed-files@v45 + with: + since_last_remote_commit: true - name: List all changed files env: diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index 6fe23699..4633d636 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t22", + name: "alloy-t23", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From a69794977ca7756108617eafacaac19496e5eb39 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 16:47:52 +0200 Subject: [PATCH 83/88] Fixed workflow --- .github/workflows/docker-build.yml | 8 ++++++-- ethereum/ethereum-reth/reth.rhai | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 8ca5021f..be5fa90e 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -174,7 +174,9 @@ jobs: build-clients: needs: [detect-changes, build-base] - if: needs.build-base.result != 'failure' && fromJson(needs.detect-changes.outputs.clients_matrix).include[0] + if: | + fromJson(needs.detect-changes.outputs.clients_matrix).include[0] && + (needs.build-base.result == 'success' || needs.build-base.result == 'skipped' || needs.build-base.result == null) runs-on: self-hosted strategy: matrix: ${{ fromJson(needs.detect-changes.outputs.clients_matrix) }} @@ -259,7 +261,9 @@ jobs: build-protocols: needs: [detect-changes, build-clients] - if: needs.build-clients.result != 'failure' && fromJson(needs.detect-changes.outputs.protocols_matrix).include[0] + if: | + fromJson(needs.detect-changes.outputs.protocols_matrix).include[0] && + (needs.build-clients.result == 'success' || needs.build-clients.result == 'skipped' || needs.build-clients.result == null) runs-on: self-hosted strategy: matrix: ${{ fromJson(needs.detect-changes.outputs.protocols_matrix) }} diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index 4633d636..a0560eb2 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t23", + name: "alloy-t24", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From ba98ef362ac7869ef3f711f045b9c40bda226ce8 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 16:51:02 +0200 Subject: [PATCH 84/88] Fixed workflow --- .github/workflows/docker-build.yml | 6 ++---- ethereum/ethereum-reth/reth.rhai | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index be5fa90e..14c71894 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -260,10 +260,8 @@ jobs: tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} build-protocols: - needs: [detect-changes, build-clients] - if: | - fromJson(needs.detect-changes.outputs.protocols_matrix).include[0] && - (needs.build-clients.result == 'success' || needs.build-clients.result == 'skipped' || needs.build-clients.result == null) + needs: [detect-changes] + if: fromJson(needs.detect-changes.outputs.protocols_matrix).include[0] runs-on: self-hosted strategy: matrix: ${{ fromJson(needs.detect-changes.outputs.protocols_matrix) }} diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index a0560eb2..7b012705 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t24", + name: "alloy-t25", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From ada7d4f86ae741f15b179d54abc2bd631f49f5a4 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 16:56:04 +0200 Subject: [PATCH 85/88] Fixed workflow --- ethereum/ethereum-erigon/Dockerfile | 7 ++++--- ethereum/ethereum-reth/Dockerfile | 13 +++++++------ ethereum/ethereum-reth/reth.rhai | 2 +- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/ethereum/ethereum-erigon/Dockerfile b/ethereum/ethereum-erigon/Dockerfile index 9b831da5..65cc50ea 100644 --- a/ethereum/ethereum-erigon/Dockerfile +++ b/ethereum/ethereum-erigon/Dockerfile @@ -1,9 +1,6 @@ ARG ERIGON_IMAGE=ghcr.io/blockjoy/erigon:latest ARG LIGHTHOUSE_IMAGE=ghcr.io/blockjoy/lighthouse:latest ARG BASE_IMAGE=ghcr.io/blockjoy/node-base:latest -ARG CLOUDFLARE_API_KEY -ARG GRAFANA_LOKI_BASICAUTH -ARG GRAFANA_PROM_BASICAUTH FROM ${ERIGON_IMAGE} as erigon-build @@ -11,6 +8,10 @@ FROM ${LIGHTHOUSE_IMAGE} AS lighthouse-build FROM ${BASE_IMAGE} +ARG CLOUDFLARE_API_KEY +ARG GRAFANA_LOKI_BASICAUTH +ARG GRAFANA_PROM_BASICAUTH + ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} diff --git a/ethereum/ethereum-reth/Dockerfile b/ethereum/ethereum-reth/Dockerfile index 859c6db7..27edf0a8 100644 --- a/ethereum/ethereum-reth/Dockerfile +++ b/ethereum/ethereum-reth/Dockerfile @@ -1,6 +1,13 @@ ARG RETH_IMAGE=ghcr.io/blockjoy/reth:latest ARG LIGHTHOUSE_IMAGE=ghcr.io/blockjoy/lighthouse:latest ARG BASE_IMAGE=ghcr.io/blockjoy/node-base:latest + +FROM ${RETH_IMAGE} AS reth-build + +FROM ${LIGHTHOUSE_IMAGE} AS lighthouse-build + +FROM ${BASE_IMAGE} + ARG CLOUDFLARE_API_KEY ARG GRAFANA_LOKI_BASICAUTH ARG GRAFANA_PROM_BASICAUTH @@ -9,12 +16,6 @@ ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} -FROM ${RETH_IMAGE} AS reth-build - -FROM ${LIGHTHOUSE_IMAGE} AS lighthouse-build - -FROM ${BASE_IMAGE} - RUN mkdir -p /root/bin COPY --from=reth-build /root/bin/reth /root/bin/ COPY --from=lighthouse-build /root/bin/lighthouse /root/bin/ diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index 7b012705..c6b08126 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t25", + name: "alloy-t26", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From 0f766958a76ba115a199d57255b228625fef5409 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 17:25:24 +0200 Subject: [PATCH 86/88] Fixed workflow --- ethereum/ethereum-reth/reth.rhai | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index c6b08126..5e2871d8 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t26", + name: "alloy-t27", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From ae4a902071cdaf1ab5fc758b572a3276a4c8d9f4 Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 17:40:33 +0200 Subject: [PATCH 87/88] Fixed workflow --- .github/workflows/docker-build.yml | 7 ++++--- ethereum/ethereum-reth/reth.rhai | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 14c71894..bc978e78 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -319,11 +319,12 @@ jobs: - name: Generate build args id: build-args run: | - # Extract all *_IMAGE args from Dockerfile and set them with SHA tags + # Extract all *_IMAGE args from Dockerfile with their specified tags IMAGE_ARGS=$(grep "^ARG.*_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do var_name=$(echo "$line" | cut -d'=' -f1 | cut -d' ' -f2) - image_name=$(echo "$line" | cut -d'=' -f2 | cut -d':' -f1 | xargs basename) - echo "${var_name}=ghcr.io/blockjoy/${image_name}:${{ steps.version.outputs.sha }}" + # Use the full image reference from the Dockerfile, including its tag + image_ref=$(echo "$line" | cut -d'=' -f2) + echo "${var_name}=${image_ref}" done | tr '\n' '\n') echo "image_args<> $GITHUB_OUTPUT echo "$IMAGE_ARGS" >> $GITHUB_OUTPUT diff --git a/ethereum/ethereum-reth/reth.rhai b/ethereum/ethereum-reth/reth.rhai index 5e2871d8..d47ebfca 100644 --- a/ethereum/ethereum-reth/reth.rhai +++ b/ethereum/ethereum-reth/reth.rhai @@ -27,7 +27,7 @@ fn base_config(metrics_port, rpc_port, ws_port, caddy_dir) { ], services: [ #{ - name: "alloy-t27", + name: "alloy", run_sh: "/usr/bin/alloy run --server.http.listen-addr=127.0.0.1:12346 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" }, #{ From 9099d795c4810b4cacd273c3b32d577171c005dd Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 17:50:58 +0200 Subject: [PATCH 88/88] Rennovate config --- renovate.json | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/renovate.json b/renovate.json index 1574bcfa..4756f0e8 100644 --- a/renovate.json +++ b/renovate.json @@ -15,33 +15,19 @@ "enabled": true }, { - "description": "Base image updates", - "matchPaths": ["node-base/Dockerfile"], - "groupName": "base-dependencies", - "separateMinorPatch": true - }, - { - "description": "Client image updates", - "matchPaths": ["clients/**/Dockerfile"], - "matchPackagePatterns": ["ghcr.io/blockjoy/node-base"], - "automerge": true, - "automergeType": "branch", - "groupName": "client-dependencies" - }, - { - "description": "Protocol implementation updates", - "matchPaths": ["ethereum/**/Dockerfile"], - "matchPackagePatterns": ["ghcr.io/blockjoy/blockjoy-*"], + "description": "All internal image updates", + "matchPaths": ["**/Dockerfile"], + "matchPackagePatterns": ["ghcr.io/blockjoy/*"], "automerge": true, "automergeType": "branch", - "groupName": "protocol-dependencies" + "groupName": "docker-dependencies" } ], "regexManagers": [ { "fileMatch": ["^.+/Dockerfile$"], "matchStrings": [ - "FROM ghcr.io/blockjoy/(?[^:]+):(?(?:[0-9]+\\.[0-9]+\\.[0-9]+-)?[a-f0-9]+)" + "ARG .*_IMAGE=ghcr.io/blockjoy/(?[^:]+):(?[^\\s]+)" ], "datasourceTemplate": "docker", "registryUrlTemplate": "https://ghcr.io"