Skip to content

Commit

Permalink
cci-stats: Add project (#115)
Browse files Browse the repository at this point in the history
* cci-stats: Add project

* Add CI

* fix lint in proxyd

* bump golangci-lint timeout
  • Loading branch information
mslipper authored Jan 8, 2025
1 parent 8e5f685 commit e191c2a
Show file tree
Hide file tree
Showing 15 changed files with 869 additions and 7 deletions.
1 change: 1 addition & 0 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ workflows:
peer-mgmt-service/.* run-build-pms true
op-ufm/.* run-build-op-ufm true
proxyd/.* run-build-proxyd true
cci-stats/.* run-build-cci-stats true
.circleci/.* run-all true
.github/.* run-all true
filters:
Expand Down
33 changes: 28 additions & 5 deletions .circleci/continue_config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ parameters:
run-build-proxyd:
type: boolean
default: false
run-build-cci-stats:
type: boolean
default: false
run-all:
type: boolean
default: false
Expand Down Expand Up @@ -288,7 +291,7 @@ jobs:
description: Go Module Name
type: string
docker:
- image: cimg/go:1.21
- image: cimg/go:1.23
steps:
- checkout
- run:
Expand All @@ -305,9 +308,9 @@ jobs:
name: run lint
command: |
if [ -f .golangci.yml ]; then
golangci-lint run -c .golangci.yml -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint -e "errors.As" -e "errors.Is" --timeout "3m0s" ./...
golangci-lint run -c .golangci.yml -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint -e "errors.As" -e "errors.Is" --timeout "5m0s" ./...
else
golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint -e "errors.As" -e "errors.Is" --timeout "3m0s" ./...
golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint -e "errors.As" -e "errors.Is" --timeout "5m0s" ./...
fi
working_directory: <<parameters.module>>

Expand Down Expand Up @@ -490,19 +493,35 @@ workflows:
docker_name: proxyd
docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
docker_context: .
cci-stats:
when:
or: [ << pipeline.parameters.run-build-cci-stats >>, << pipeline.parameters.run-all >> ]
jobs:
- go-lint:
name: cci-stats-lint
module: cci-stats
- go-test:
name: cci-stats-tests
module: cci-stats
- docker-build:
name: cci-stats-docker-build
docker_file: cci-stats/Dockerfile
docker_name: cci-stats
docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
docker_context: .
release:
jobs:
- log-config-results:
filters:
tags:
only: /^(peer-mgmt-service|proxyd|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/
only: /^(cci-stats|peer-mgmt-service|proxyd|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/
branches:
ignore: /.*/
- hold:
type: approval
filters:
tags:
only: /^(peer-mgmt-service|proxyd|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/
only: /^(cci-stats|peer-mgmt-service|proxyd|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/
branches:
ignore: /.*/
- docker-build:
Expand All @@ -515,6 +534,7 @@ workflows:
- proxyd
- op-conductor-mon
- peer-mgmt-service
- cci-stats
name: <<matrix.docker_name>>-docker-build
filters:
tags:
Expand All @@ -536,6 +556,7 @@ workflows:
- proxyd
- op-conductor-mon
- peer-mgmt-service
- cci-stats
name: <<matrix.docker_name>>-docker-publish
filters:
tags:
Expand All @@ -555,6 +576,7 @@ workflows:
- proxyd
- op-conductor-mon
- peer-mgmt-service
- cci-stats
name: <<matrix.docker_name>>-docker-tag
filters:
tags:
Expand All @@ -575,6 +597,7 @@ workflows:
- proxyd
- op-conductor-mon
- peer-mgmt-service
- cci-stats
name: <<matrix.module>>-go-release
filters:
tags:
Expand Down
56 changes: 56 additions & 0 deletions cci-stats/.goreleaser.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
# yaml-language-server: $schema=https://goreleaser.com/static/schema-pro.json
# vim: set ts=2 sw=2 tw=0 fo=cnqoj

version: 2

project_name: cci-stats

before:
hooks:
# You may remove this if you don't use go modules.
- go mod tidy

builds:
- id: main
main: ./cmd/runner
binary: cci-stats
goos:
- linux
- windows
- darwin
goarch:
- amd64
- arm64
ignore:
- goos: windows
goarch: arm64
- goos: linux
goarch: arm64
mod_timestamp: "{{ .CommitTimestamp }}"
ldflags:
- -X main.GitCommit={{ .FullCommit }}
- -X main.GitDate={{ .CommitDate }}
- -X main.Version={{ .Version }}

archives:
- format: tar.gz
# this name template makes the OS and Arch compatible with the results of `uname`.
name_template: "{{ .ProjectName }}-{{.Version}}-{{ tolower .Os }}-{{ .Arch }}"
# use zip for windows archives
wrap_in_directory: true
format_overrides:
- goos: windows
format: zip

changelog:
sort: asc

release:
github:
owner: ethereum-optimism
name: infra
make_latest: false

monorepo:
tag_prefix: cci-stats/
dir: cci-stats
20 changes: 20 additions & 0 deletions cci-stats/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
FROM golang:1.23.4-alpine3.20 as builder

ARG GITCOMMIT=docker
ARG GITDATE=docker
ARG GITVERSION=docker

COPY ./cci-stats /app

WORKDIR /app
RUN apk add --no-cache make jq bash git alpine-sdk
RUN make build

FROM alpine:3.20
RUN addgroup -S app && adduser -S app -G app
USER app
WORKDIR /app

COPY --from=builder /app/bin/cci-stats /app

ENTRYPOINT ["/app/cci-stats"]
28 changes: 28 additions & 0 deletions cci-stats/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
LDFLAGSSTRING +=-X main.GitCommit=$(GITCOMMIT)
LDFLAGSSTRING +=-X main.GitDate=$(GITDATE)
LDFLAGSSTRING +=-X main.GitVersion=$(GITVERSION)
LDFLAGS := -ldflags "$(LDFLAGSSTRING)"

all: build

docker:
docker build ../ -f Dockerfile -t cci-stats:latest

build:
env GO111MODULE=on go build -v $(LDFLAGS) -o ./bin/cci-stats ./cmd/runner

clean:
rm ./bin/cci-stats

test:
go test -v ./...

lint:
golangci-lint run ./...

.PHONY: \
build \
clean \
test \
lint \
docker
18 changes: 18 additions & 0 deletions cci-stats/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# cci-stats

`cci-stats` is a program that reads stats from the CircleCI API, and writes them to a Postgres database. It is used
internally at OP Labs to keep track of CI pass rate, merge throughput, and other engineering health metrics.

To run the program, specify the following env vars:

- `CCI_KEY`: CircleCI API Key
- `DATABASE_URI`: Postgres database URI
- `PROJECT_SLUG`: Slug of the CCI project you want to grab stats for
- `BRANCH_PATTERN`: Regex pattern to filter branches by
- `WORKFLOW_PATTERN`: Regex pattern to filter workflows by
- `FETCH_LIMIT_DAYS`: Maximum number of days to look into the past for new build data
- `MAX_CONCURRENT_FETCH_JOBS`: How many concurrent requests to CCI to make at once. Used to tune rate limits
- `SLOW_TEST_THRESHOLD_SECONDS`: Tests slower than this threshold are written to the database as "slow tests" for
further debugging

Then run `go run cmd/runner/main.go`.
98 changes: 98 additions & 0 deletions cci-stats/cmd/runner/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package main

import (
"context"
"fmt"
"os"
"os/signal"
"regexp"
"strconv"

"github.com/axelKingsley/go-circleci"
"github.com/ethereum-optimism/infra/cci-stats/pkg/config"
"github.com/ethereum-optimism/infra/cci-stats/pkg/db"
"github.com/ethereum-optimism/infra/cci-stats/pkg/service"
)

func main() {
if err := run(); err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1)
}
}

func run() error {
cciKey := requiredStrEnv("CCI_KEY")
dbURI := requiredStrEnv("DATABASE_URI")
projectSlug := requiredStrEnv("PROJECT_SLUG")
branchPattern := requiredStrEnv("BRANCH_PATTERN")
workflowPattern := requiredStrEnv("WORKFLOW_PATTERN")
fetchLimitDays := requiredIntEnv("FETCH_LIMIT_DAYS")
maxConcurrentFetchJobs := requiredIntEnv("MAX_CONCURRENT_FETCH_JOBS")
slowTestThresholdSeconds := requiredIntEnv("SLOW_TEST_THRESHOLD_SECONDS")

ctx, cancel := context.WithCancel(context.Background())
defer cancel()

dbConn, err := db.New(ctx, dbURI)
if err != nil {
return fmt.Errorf("failed to connect to db: %w", err)
}
defer dbConn.Close()

cfg := config.Config{
ProjectSlug: projectSlug,
BranchPatternRegex: regexp.MustCompile(branchPattern),
WorkflowPatternRegex: regexp.MustCompile(workflowPattern),
FetchLimitDays: fetchLimitDays,
MaxConcurrentFetchJobs: maxConcurrentFetchJobs,
SlowTestThresholdSeconds: float64(slowTestThresholdSeconds),
}

clientCfg := circleci.DefaultConfig()
clientCfg.Token = cciKey
client, err := circleci.NewClient(clientCfg)
if err != nil {
return fmt.Errorf("failed to create circleci client: %w", err)
}

errC := make(chan error)
go func() {
err := service.GenerateReport(ctx, cfg, client, dbConn)
errC <- err
}()

sigs := make(chan os.Signal, 1)
signal.Notify(sigs, os.Interrupt)

for {
select {
case <-sigs:
cancel()
case err := <-errC:
return err
case <-ctx.Done():
return nil
}
}
}

func requiredStrEnv(envVar string) string {
val := os.Getenv(envVar)
if val == "" {
panic(fmt.Errorf("%s must be set", envVar))
}
return val
}

func requiredIntEnv(envVar string) int {
val := os.Getenv(envVar)
if val == "" {
panic(fmt.Errorf("%s must be set", envVar))
}
out, err := strconv.Atoi(val)
if err != nil {
panic(fmt.Errorf("%s must be an integer", envVar))
}
return out
}
21 changes: 21 additions & 0 deletions cci-stats/go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
module github.com/ethereum-optimism/infra/cci-stats

go 1.23.1

require (
github.com/axelKingsley/go-circleci v0.9.1
github.com/jackc/pgx/v5 v5.7.1
github.com/sourcegraph/conc v0.3.0
)

require (
github.com/google/go-querystring v1.1.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.9.0 // indirect
golang.org/x/crypto v0.28.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/text v0.19.0 // indirect
)
42 changes: 42 additions & 0 deletions cci-stats/go.sum
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
github.com/axelKingsley/go-circleci v0.9.1 h1:TsGXu2QeTaUO47BKVw9fBTKCH9tYiLltztFxTxE1GYA=
github.com/axelKingsley/go-circleci v0.9.1/go.mod h1:fk2iqm3nLWa+Xj9wWUpnfDnuUdLizC6Rj8xEUxj6E18=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs=
github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
Loading

0 comments on commit e191c2a

Please sign in to comment.