generated from ethereum-optimism/.github
-
Notifications
You must be signed in to change notification settings - Fork 33
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* cci-stats: Add project * Add CI * fix lint in proxyd * bump golangci-lint timeout
- Loading branch information
Showing
15 changed files
with
869 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,56 @@ | ||
# yaml-language-server: $schema=https://goreleaser.com/static/schema-pro.json | ||
# vim: set ts=2 sw=2 tw=0 fo=cnqoj | ||
|
||
version: 2 | ||
|
||
project_name: cci-stats | ||
|
||
before: | ||
hooks: | ||
# You may remove this if you don't use go modules. | ||
- go mod tidy | ||
|
||
builds: | ||
- id: main | ||
main: ./cmd/runner | ||
binary: cci-stats | ||
goos: | ||
- linux | ||
- windows | ||
- darwin | ||
goarch: | ||
- amd64 | ||
- arm64 | ||
ignore: | ||
- goos: windows | ||
goarch: arm64 | ||
- goos: linux | ||
goarch: arm64 | ||
mod_timestamp: "{{ .CommitTimestamp }}" | ||
ldflags: | ||
- -X main.GitCommit={{ .FullCommit }} | ||
- -X main.GitDate={{ .CommitDate }} | ||
- -X main.Version={{ .Version }} | ||
|
||
archives: | ||
- format: tar.gz | ||
# this name template makes the OS and Arch compatible with the results of `uname`. | ||
name_template: "{{ .ProjectName }}-{{.Version}}-{{ tolower .Os }}-{{ .Arch }}" | ||
# use zip for windows archives | ||
wrap_in_directory: true | ||
format_overrides: | ||
- goos: windows | ||
format: zip | ||
|
||
changelog: | ||
sort: asc | ||
|
||
release: | ||
github: | ||
owner: ethereum-optimism | ||
name: infra | ||
make_latest: false | ||
|
||
monorepo: | ||
tag_prefix: cci-stats/ | ||
dir: cci-stats |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
FROM golang:1.23.4-alpine3.20 as builder | ||
|
||
ARG GITCOMMIT=docker | ||
ARG GITDATE=docker | ||
ARG GITVERSION=docker | ||
|
||
COPY ./cci-stats /app | ||
|
||
WORKDIR /app | ||
RUN apk add --no-cache make jq bash git alpine-sdk | ||
RUN make build | ||
|
||
FROM alpine:3.20 | ||
RUN addgroup -S app && adduser -S app -G app | ||
USER app | ||
WORKDIR /app | ||
|
||
COPY --from=builder /app/bin/cci-stats /app | ||
|
||
ENTRYPOINT ["/app/cci-stats"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
LDFLAGSSTRING +=-X main.GitCommit=$(GITCOMMIT) | ||
LDFLAGSSTRING +=-X main.GitDate=$(GITDATE) | ||
LDFLAGSSTRING +=-X main.GitVersion=$(GITVERSION) | ||
LDFLAGS := -ldflags "$(LDFLAGSSTRING)" | ||
|
||
all: build | ||
|
||
docker: | ||
docker build ../ -f Dockerfile -t cci-stats:latest | ||
|
||
build: | ||
env GO111MODULE=on go build -v $(LDFLAGS) -o ./bin/cci-stats ./cmd/runner | ||
|
||
clean: | ||
rm ./bin/cci-stats | ||
|
||
test: | ||
go test -v ./... | ||
|
||
lint: | ||
golangci-lint run ./... | ||
|
||
.PHONY: \ | ||
build \ | ||
clean \ | ||
test \ | ||
lint \ | ||
docker |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
# cci-stats | ||
|
||
`cci-stats` is a program that reads stats from the CircleCI API, and writes them to a Postgres database. It is used | ||
internally at OP Labs to keep track of CI pass rate, merge throughput, and other engineering health metrics. | ||
|
||
To run the program, specify the following env vars: | ||
|
||
- `CCI_KEY`: CircleCI API Key | ||
- `DATABASE_URI`: Postgres database URI | ||
- `PROJECT_SLUG`: Slug of the CCI project you want to grab stats for | ||
- `BRANCH_PATTERN`: Regex pattern to filter branches by | ||
- `WORKFLOW_PATTERN`: Regex pattern to filter workflows by | ||
- `FETCH_LIMIT_DAYS`: Maximum number of days to look into the past for new build data | ||
- `MAX_CONCURRENT_FETCH_JOBS`: How many concurrent requests to CCI to make at once. Used to tune rate limits | ||
- `SLOW_TEST_THRESHOLD_SECONDS`: Tests slower than this threshold are written to the database as "slow tests" for | ||
further debugging | ||
|
||
Then run `go run cmd/runner/main.go`. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,98 @@ | ||
package main | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"os" | ||
"os/signal" | ||
"regexp" | ||
"strconv" | ||
|
||
"github.com/axelKingsley/go-circleci" | ||
"github.com/ethereum-optimism/infra/cci-stats/pkg/config" | ||
"github.com/ethereum-optimism/infra/cci-stats/pkg/db" | ||
"github.com/ethereum-optimism/infra/cci-stats/pkg/service" | ||
) | ||
|
||
func main() { | ||
if err := run(); err != nil { | ||
fmt.Fprintf(os.Stderr, "error: %v\n", err) | ||
os.Exit(1) | ||
} | ||
} | ||
|
||
func run() error { | ||
cciKey := requiredStrEnv("CCI_KEY") | ||
dbURI := requiredStrEnv("DATABASE_URI") | ||
projectSlug := requiredStrEnv("PROJECT_SLUG") | ||
branchPattern := requiredStrEnv("BRANCH_PATTERN") | ||
workflowPattern := requiredStrEnv("WORKFLOW_PATTERN") | ||
fetchLimitDays := requiredIntEnv("FETCH_LIMIT_DAYS") | ||
maxConcurrentFetchJobs := requiredIntEnv("MAX_CONCURRENT_FETCH_JOBS") | ||
slowTestThresholdSeconds := requiredIntEnv("SLOW_TEST_THRESHOLD_SECONDS") | ||
|
||
ctx, cancel := context.WithCancel(context.Background()) | ||
defer cancel() | ||
|
||
dbConn, err := db.New(ctx, dbURI) | ||
if err != nil { | ||
return fmt.Errorf("failed to connect to db: %w", err) | ||
} | ||
defer dbConn.Close() | ||
|
||
cfg := config.Config{ | ||
ProjectSlug: projectSlug, | ||
BranchPatternRegex: regexp.MustCompile(branchPattern), | ||
WorkflowPatternRegex: regexp.MustCompile(workflowPattern), | ||
FetchLimitDays: fetchLimitDays, | ||
MaxConcurrentFetchJobs: maxConcurrentFetchJobs, | ||
SlowTestThresholdSeconds: float64(slowTestThresholdSeconds), | ||
} | ||
|
||
clientCfg := circleci.DefaultConfig() | ||
clientCfg.Token = cciKey | ||
client, err := circleci.NewClient(clientCfg) | ||
if err != nil { | ||
return fmt.Errorf("failed to create circleci client: %w", err) | ||
} | ||
|
||
errC := make(chan error) | ||
go func() { | ||
err := service.GenerateReport(ctx, cfg, client, dbConn) | ||
errC <- err | ||
}() | ||
|
||
sigs := make(chan os.Signal, 1) | ||
signal.Notify(sigs, os.Interrupt) | ||
|
||
for { | ||
select { | ||
case <-sigs: | ||
cancel() | ||
case err := <-errC: | ||
return err | ||
case <-ctx.Done(): | ||
return nil | ||
} | ||
} | ||
} | ||
|
||
func requiredStrEnv(envVar string) string { | ||
val := os.Getenv(envVar) | ||
if val == "" { | ||
panic(fmt.Errorf("%s must be set", envVar)) | ||
} | ||
return val | ||
} | ||
|
||
func requiredIntEnv(envVar string) int { | ||
val := os.Getenv(envVar) | ||
if val == "" { | ||
panic(fmt.Errorf("%s must be set", envVar)) | ||
} | ||
out, err := strconv.Atoi(val) | ||
if err != nil { | ||
panic(fmt.Errorf("%s must be an integer", envVar)) | ||
} | ||
return out | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
module github.com/ethereum-optimism/infra/cci-stats | ||
|
||
go 1.23.1 | ||
|
||
require ( | ||
github.com/axelKingsley/go-circleci v0.9.1 | ||
github.com/jackc/pgx/v5 v5.7.1 | ||
github.com/sourcegraph/conc v0.3.0 | ||
) | ||
|
||
require ( | ||
github.com/google/go-querystring v1.1.0 // indirect | ||
github.com/jackc/pgpassfile v1.0.0 // indirect | ||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect | ||
github.com/jackc/puddle/v2 v2.2.2 // indirect | ||
go.uber.org/atomic v1.7.0 // indirect | ||
go.uber.org/multierr v1.9.0 // indirect | ||
golang.org/x/crypto v0.28.0 // indirect | ||
golang.org/x/sync v0.8.0 // indirect | ||
golang.org/x/text v0.19.0 // indirect | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
github.com/axelKingsley/go-circleci v0.9.1 h1:TsGXu2QeTaUO47BKVw9fBTKCH9tYiLltztFxTxE1GYA= | ||
github.com/axelKingsley/go-circleci v0.9.1/go.mod h1:fk2iqm3nLWa+Xj9wWUpnfDnuUdLizC6Rj8xEUxj6E18= | ||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= | ||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= | ||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= | ||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= | ||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= | ||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= | ||
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= | ||
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= | ||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= | ||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= | ||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= | ||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= | ||
github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs= | ||
github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA= | ||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= | ||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= | ||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= | ||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= | ||
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= | ||
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= | ||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= | ||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= | ||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= | ||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= | ||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= | ||
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= | ||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= | ||
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= | ||
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= | ||
golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= | ||
golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= | ||
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= | ||
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= | ||
golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= | ||
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= | ||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= | ||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= | ||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= | ||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= | ||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= |
Oops, something went wrong.