Skip to content

Commit

Permalink
support triggering jenkins jobs
Browse files Browse the repository at this point in the history 
Add support for trigggering CI (Jenkins) Jobs for PRs in the auoupdater
queue.
CI jobs are triggered when a PR is placed first in the queue and it's
check status is pending and when the branch of a PR changes.

The URL of the Jenkins CI server and the jobs that should be run are configured
in the config.toml file. The jobs are triggered by sending HTTP-Requests
to the URL of the job. It is supported to pass parameters to jobs.
Triggering jobs is retried automatically on errors.

The webinterface displays the CI configuration and configuration
settings are now shown in bold.

The main function has been refactored to:
 - start listening for github web hook events before syncing the
   autoupdater, to ensure events are received and processed that
   happened while a PR is added to the queue,
 - call autoupdater.Stop on shutdown
  • Loading branch information
@fho
fho committed Dec 2, 2024
1 parent 9ca0e51 commit ad9513f
Show file tree
Hide file tree
Showing 27 changed files with 1,160 additions and 492 deletions.
1 change: 1 addition & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ gen_mocks:
$(info * generating mock code)
mockgen -package mocks -source internal/autoupdate/autoupdate.go -destination internal/autoupdate/mocks/autoupdate.go
mockgen -package mocks -source internal/githubclt/client.go -destination internal/autoupdate/mocks/githubclient.go
mockgen -package mocks -destination internal/autoupdate/mocks/ciclient.go github.com/simplesurance/directorius/internal/autoupdate CIClient

.PHONY: check
check:
Expand Down
165 changes: 96 additions & 69 deletions cmd/directorius/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"errors"
"flag"
"fmt"
"math"
"net/http"
"os"
"time"
Expand All @@ -13,8 +14,10 @@ import (
"github.com/simplesurance/directorius/internal/cfg"
"github.com/simplesurance/directorius/internal/githubclt"
"github.com/simplesurance/directorius/internal/goordinator"
"github.com/simplesurance/directorius/internal/jenkins"
"github.com/simplesurance/directorius/internal/logfields"
"github.com/simplesurance/directorius/internal/provider/github"
"github.com/simplesurance/directorius/internal/set"

"github.com/spf13/pflag"
zaplogfmt "github.com/sykesm/zap-logfmt"
Expand All @@ -32,7 +35,7 @@ const (

var logger *zap.Logger

// Version is set via a ldflag on compilation
// Version is set by goreleaser
var Version = "version unknown"

const EventChannelBufferSize = 1024
Expand All @@ -48,7 +51,7 @@ func exitOnErr(msg string, err error) {

func panicHandler() {
if r := recover(); r != nil {
logger.Info(
logger.Error(
"panic caught, terminating gracefully",
zap.String("panic", fmt.Sprintf("%v", r)),
zap.StackSkip("stacktrace", 1),
Expand Down Expand Up @@ -283,6 +286,7 @@ func mustInitLogger(config *cfg.Config) {

logger = logger.Named("main")
zap.ReplaceGlobals(logger)
// TODO: call goodbye.Exit on Fatal and Panic calls (zap.WithFatalHook)

goodbye.Register(func(context.Context, os.Signal) {
if err := logger.Sync(); err != nil {
Expand All @@ -307,32 +311,34 @@ func normalizeHTTPEndpoint(endpoint string) string {
return endpoint
}

func mustStartPullRequestAutoupdater(config *cfg.Config, githubClient *githubclt.Client, mux *http.ServeMux) (*autoupdate.Autoupdater, chan<- *github.Event) {
if !config.TriggerOnAutoMerge && len(config.TriggerOnLabels) == 0 {
fmt.Fprintf(os.Stderr, "ERROR: config file: %s: trigger_on_auto_merge must be true or trigger_labels must be defined, both are empty in the configuration file\n", *args.ConfigFile)
os.Exit(1)
func mustConfigCItoAutoupdaterCI(cfg *cfg.CI) *autoupdate.CI {
if cfg == nil {
return nil
}

if len(config.HeadLabel) == 0 {
fmt.Fprintf(os.Stderr, "ERROR: config file %s: queue_pr_head_label must be provided when autoupdater is enabled\n", *args.ConfigFile)
if len(cfg.Jobs) > 0 && cfg.ServerURL == "" {
fmt.Fprintf(os.Stderr, "ERROR: config file: %s: ci.jobs are defined but ci.server_url is empty\n", *args.ConfigFile)
os.Exit(1)
}

if len(config.GithubAPIToken) == 0 {
fmt.Fprintf(os.Stderr, "ERROR: config file %s: github_api_token must be provided when autoupdater is enabled\n", *args.ConfigFile)
if cfg.ServerURL != "" && len(cfg.Jobs) == 0 {
fmt.Fprintf(os.Stderr, "ERROR: config file: %s: ci.server_url is defined but no ci.jobs are defined\n", *args.ConfigFile)
os.Exit(1)
}

if len(config.HTTPGithubWebhookEndpoint) == 0 {
fmt.Fprintf(os.Stderr, "ERROR: config file %s: github_webhook_endpoint must be provided when autoupdater is enabled\n", *args.ConfigFile)
os.Exit(1)
}

if len(config.Repositories) == 0 {
logger.Info("github pull request updater is disabled, autoupdater.repository config field is empty")
return nil, nil
var result autoupdate.CI
result.Client = jenkins.NewClient(cfg.ServerURL, cfg.BasicAuth.User, cfg.BasicAuth.Password)
for _, job := range cfg.Jobs {
result.Jobs = append(result.Jobs, &jenkins.JobTemplate{
RelURL: job.Endpoint,
Parameters: job.Parameters,
})
}
return &result
}

func mustStartPullRequestAutoupdater(config *cfg.Config, ch chan *github.Event, githubClient *githubclt.Client, mux *http.ServeMux) *autoupdate.Autoupdater {
repos := make([]autoupdate.Repository, 0, len(config.Repositories))
for _, r := range config.Repositories {
repos = append(repos, autoupdate.Repository{
Expand All @@ -341,18 +347,30 @@ func mustStartPullRequestAutoupdater(config *cfg.Config, githubClient *githubclt
})
}

ch := make(chan *github.Event, EventChannelBufferSize)

autoupdater := autoupdate.NewAutoupdater(
githubClient,
ch,
goordinator.NewRetryer(),
repos,
config.TriggerOnAutoMerge,
config.TriggerOnLabels,
config.HeadLabel,
autoupdate.DryRun(*args.DryRun),
autoupdate.Config{
GitHubClient: githubClient,
EventChan: ch,
Retryer: goordinator.NewRetryer(),
MonitoredRepositories: set.From(repos),
TriggerOnAutomerge: config.TriggerOnAutoMerge,
TriggerLabels: set.From(config.TriggerOnLabels),
HeadLabel: config.HeadLabel,
DryRun: *args.DryRun,
CI: mustConfigCItoAutoupdaterCI(&config.CI),
},
)

ctx, cancelFn := context.WithTimeout(context.Background(), 15*time.Minute)
defer cancelFn()
if err := autoupdater.InitSync(ctx); err != nil {
logger.Error(
"autoupdater: initial synchronization failed",
logfields.Event("autoupdate_initial_sync_failed"),
zap.Error(err),
)
}

autoupdater.Start()

if config.WebInterfaceEndpoint != "" {
Expand All @@ -364,13 +382,43 @@ func mustStartPullRequestAutoupdater(config *cfg.Config, githubClient *githubclt
)
}

return autoupdater, ch
return autoupdater
}

func mustValidateConfig(config *cfg.Config) {
if config.HTTPListenAddr == "" && config.HTTPSListenAddr == "" {
fmt.Fprintf(os.Stderr, "https_server_listen_addr or http_server_listen_addr must be defined in the config file, both are unset")
os.Exit(1)
}

if !config.TriggerOnAutoMerge && len(config.TriggerOnLabels) == 0 {
fmt.Fprintf(os.Stderr, "ERROR: config file: %s: trigger_on_auto_merge must be true or trigger_labels must be defined, both are empty in the configuration file\n", *args.ConfigFile)
os.Exit(1)
}

if len(config.HeadLabel) == 0 {
fmt.Fprintf(os.Stderr, "ERROR: config file %s: queue_pr_head_label must be provided when autoupdater is enabled\n", *args.ConfigFile)
os.Exit(1)
}

if len(config.GithubAPIToken) == 0 {
fmt.Fprintf(os.Stderr, "ERROR: config file %s: github_api_token must be provided when autoupdater is enabled\n", *args.ConfigFile)
os.Exit(1)
}

if len(config.HTTPGithubWebhookEndpoint) == 0 {
fmt.Fprintf(os.Stderr, "ERROR: config file %s: github_webhook_endpoint must be provided when autoupdater is enabled\n", *args.ConfigFile)
os.Exit(1)
}

if len(config.Repositories) == 0 {
logger.Info("github pull request updater is disabled, autoupdater.repository config field is empty")
}
}

func main() {
defer panicHandler()

defer goodbye.Exit(context.Background(), 1)
goodbye.Notify(context.Background())

mustParseCommandlineParams()
Expand All @@ -381,13 +429,11 @@ func main() {
}

config := mustParseCfg()
mustValidateConfig(config)

mustInitLogger(config)

githubClient := githubclt.New(config.GithubAPIToken)

logger.Info(
"loaded cfg file",
logger.Info("loaded cfg file",
logfields.Event("cfg_loaded"),
zap.String("cfg_file", *args.ConfigFile),
zap.String("http_server_listen_addr", config.HTTPListenAddr),
Expand All @@ -403,31 +449,25 @@ func main() {
zap.Strings("trigger_labels", config.TriggerOnLabels),
zap.Any("repositories", config.Repositories),
zap.String("webinterface_endpoint", config.WebInterfaceEndpoint),
zap.String("ci.base_url", config.CI.ServerURL),
zap.String("ci.basic_auth.user", config.CI.BasicAuth.User),
zap.String("ci.basic_auth.password", hide(config.CI.BasicAuth.User)),
zap.Any("ci.job", config.CI.Jobs),
)

goodbye.Register(func(_ context.Context, sig os.Signal) {
logger.Info(fmt.Sprintf("terminating, received signal %s", sig.String()))
})

if config.HTTPListenAddr == "" && config.HTTPSListenAddr == "" {
fmt.Fprintf(os.Stderr, "https_server_listen_addr or http_server_listen_addr must be defined in the config file, both are unset")
os.Exit(1)
}
githubClient := githubclt.New(config.GithubAPIToken)

var chans []chan<- *github.Event
goodbye.RegisterWithPriority(func(_ context.Context, sig os.Signal) {
logger.Info(fmt.Sprintf("terminating, received signal %s", sig.String()))
}, math.MinInt)

mux := http.NewServeMux()

autoupdater, ch := mustStartPullRequestAutoupdater(config, githubClient, mux)
if ch != nil {
chans = append(chans, ch)
}

ch := make(chan *github.Event, EventChannelBufferSize)
gh := github.New(
chans,
[]chan<- *github.Event{ch},
github.WithPayloadSecret(config.GithubWebHookSecret),
)

mux.HandleFunc(config.HTTPGithubWebhookEndpoint, gh.HTTPHandler)
logger.Info(
"registered github webhook event http endpoint",
Expand Down Expand Up @@ -457,26 +497,13 @@ func main() {
)
}

goodbye.Register(func(context.Context, os.Signal) {
logger.Debug(
"stopping event loop",
logfields.Event("event_loop_stopping"),
)
})
autoupdater := mustStartPullRequestAutoupdater(config, ch, githubClient, mux)

if autoupdater != nil {
ctx, cancelFn := context.WithTimeout(context.Background(), 15*time.Minute)
if err := autoupdater.InitSync(ctx); err != nil {
logger.Error(
"autoupdater: initial synchronization failed",
logfields.Event("autoupdate_initial_sync_failed"),
zap.Error(err),
)
}
cancelFn()

autoupdater.Start()
}
waitForTermCh := make(chan struct{})
goodbye.RegisterWithPriority(func(context.Context, os.Signal) {
autoupdater.Stop()
close(waitForTermCh)
}, -1)

select {} // TODO: refactor this, allow clean shutdown
<-waitForTermCh
}
28 changes: 23 additions & 5 deletions config.example.toml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#http_server_listen_addr = ":8085"
https_server_listen_addr = ":8084"
https_ssl_cert_file = ""
https_ssl_key_file = ""
http_server_listen_addr = ":8085"
#https_server_listen_addr = ":8084"
#https_ssl_cert_file = ""
#https_ssl_key_file = ""

# the local http and https endpoints that receives github webhook events
github_webhook_endpoint = "/listener/github"
Expand All @@ -22,7 +22,7 @@ log_format = "logfmt"
log_time_key = "time_iso8601"

# log_level controls the priority threshold for log messages.
# All messages with the the specified or a higher priority are logged.
# All messages with the specified or a higher priority are logged.
# Supported values: debug, info, warn, error, panic, fatal
log_level = "info"

Expand All @@ -43,3 +43,21 @@ queue_pr_head_label = "autoupdater-first"
owner = "simplesurance"
repository = "goordinator"

[ci]
server_url = "https://jenkins"
basic_auth.user = "cetautomatix"
basic_auth.password = "hammer"

[[ci.job]]
endpoint = "job/check/{{ queryescape .Branch }}/build"

[[ci.job]]
endpoint = "job/build/{{ queryescape .PullRequestNr }}/build"

[[ci.job]]
endpoint = "job/test/buildWithParameters"

[ci.job.parameters]
branch = "{{ .Branch }}"
pr_nr = "{{ .PullRequestNr }}"

Loading

0 comments on commit ad9513f

Please sign in to comment.