Add azure devops webhook support

go.mod

@@ -9,6 +9,7 @@ replace (
 
 require (
 	github.com/Masterminds/semver/v3 v3.2.1
+	github.com/go-playground/webhooks/v6 v6.3.0
 	github.com/gogits/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85
 	github.com/golang/mock v1.6.0
 	github.com/gorilla/mux v1.8.0

go.sum

@@ -268,11 +268,14 @@ github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+
 github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
 github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4=
 github.com/go-openapi/validate v0.19.8/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4=
+github.com/go-playground/webhooks/v6 v6.3.0 h1:zBLUxK1Scxwi97TmZt5j/B/rLlard2zY7P77FHg58FE=
+github.com/go-playground/webhooks/v6 v6.3.0/go.mod h1:GCocmfMtpJdkEOM1uG9p2nXzg1kY5X/LtvQgtPHUaaA=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gogits/go-gogs-client v0.0.0-20200905025246-8bb8a50cb355/go.mod h1:cY2AIrMgHm6oOHmR7jY+9TtjzSjQ3iG7tURJG3Y6XH0=
 github.com/gogits/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85 h1:04sojTxgYxu1L4Hn7Tgf7UVtIosVa6CuHtvNY+7T1K4=
 github.com/gogits/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85/go.mod h1:cY2AIrMgHm6oOHmR7jY+9TtjzSjQ3iG7tURJG3Y6XH0=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=

pkg/webhook/webhook.go

@@ -8,6 +8,8 @@ import (
 	"strings"
 
 	"github.com/Masterminds/semver/v3"
+
+	"github.com/go-playground/webhooks/v6/azuredevops"
 	gogsclient "github.com/gogits/go-gogs-client"
 	"github.com/gorilla/mux"
 	v1controller "github.com/rancher/gitjob/pkg/generated/controllers/gitjob.cattle.io/v1"
@@ -47,6 +49,7 @@ type Webhook struct {
 	bitbucket       *bitbucket.Webhook
 	bitbucketServer *bitbucketserver.Webhook
 	gogs            *gogs.Webhook
+	azureDevops     *azuredevops.Webhook
 }
 
 func New(ctx context.Context, rContext *types.Context) *Webhook {
@@ -90,6 +93,10 @@ func (w *Webhook) onSecretChange(_ string, secret *corev1.Secret) (*corev1.Secre
 	if err != nil {
 		return nil, err
 	}
+	w.azureDevops, err = azuredevops.New()
+	if err != nil {
+		return nil, err
+	}
 	return nil, nil
 }
 
@@ -110,6 +117,8 @@ func (w *Webhook) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 		payload, err = w.bitbucket.Parse(r, bitbucket.RepoPushEvent)
 	case r.Header.Get("X-Event-Key") != "":
 		payload, err = w.bitbucketServer.Parse(r, bitbucketserver.RepositoryReferenceChangedEvent)
+	case r.Header.Get("X-Vss-Activityid") != "" || r.Header.Get("X-Vss-Subscriptionid") != "":
+		payload, err = w.azureDevops.Parse(r, azuredevops.GitPushEventType)
 	default:
 		logrus.Debug("Ignoring unknown webhook event")
 		return
@@ -130,14 +139,26 @@ func (w *Webhook) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 		branch, tag = getBranchTagFromRef(t.Ref)
 		revision = t.After
 		repoURLs = append(repoURLs, t.Repository.HTMLURL)
+		if err := w.processGitPushEvent(repoURLs, tag, branch, revision); err != nil {
+			logAndReturn(rw, err)
+			return
+		}
 	case gitlab.PushEventPayload:
 		branch, tag = getBranchTagFromRef(t.Ref)
 		revision = t.CheckoutSHA
 		repoURLs = append(repoURLs, t.Project.WebURL)
+		if err := w.processGitPushEvent(repoURLs, tag, branch, revision); err != nil {
+			logAndReturn(rw, err)
+			return
+		}
 	case gitlab.TagEventPayload:
 		branch, tag = getBranchTagFromRef(t.Ref)
 		revision = t.CheckoutSHA
 		repoURLs = append(repoURLs, t.Project.WebURL)
+		if err := w.processGitPushEvent(repoURLs, tag, branch, revision); err != nil {
+			logAndReturn(rw, err)
+			return
+		}
 	// https://support.atlassian.com/bitbucket-cloud/docs/event-payloads/#Push
 	case bitbucket.RepoPushPayload:
 		repoURLs = append(repoURLs, t.Repository.Links.HTML.Href)
@@ -148,7 +169,10 @@ func (w *Webhook) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 			} else if change.New.Type == "tag" {
 				tag = change.New.Name
 			}
-			break
+			if err := w.processGitPushEvent(repoURLs, tag, branch, revision); err != nil {
+				logAndReturn(rw, err)
+				return
+			}
 		}
 	case bitbucketserver.RepositoryReferenceChangedPayload:
 		for _, l := range t.Repository.Links["clone"].([]interface{}) {
@@ -163,31 +187,50 @@ func (w *Webhook) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 		for _, change := range t.Changes {
 			revision = change.ToHash
 			branch, tag = getBranchTagFromRef(change.ReferenceId)
-			break
+			if err := w.processGitPushEvent(repoURLs, tag, branch, revision); err != nil {
+				logAndReturn(rw, err)
+				return
+			}
 		}
 	case gogsclient.PushPayload:
 		repoURLs = append(repoURLs, t.Repo.HTMLURL)
 		branch, tag = getBranchTagFromRef(t.Ref)
 		revision = t.After
+		if err := w.processGitPushEvent(repoURLs, tag, branch, revision); err != nil {
+			logAndReturn(rw, err)
+			return
+		}
+	case azuredevops.GitPushEvent:
+		repoURLs = append(repoURLs, t.Resource.Repository.RemoteURL)
+		for _, refUpdate := range t.Resource.RefUpdates {
+			branch, tag = getBranchTagFromRef(refUpdate.Name)
+			revision = refUpdate.NewObjectID
+			if err := w.processGitPushEvent(repoURLs, tag, branch, revision); err != nil {
+				logAndReturn(rw, err)
+				return
+			}
+		}
 	}
 
+	rw.WriteHeader(200)
+	rw.Write([]byte("succeeded"))
+}
+
+func (w *Webhook) processGitPushEvent(repoURLs []string, tag, branch, revision string) error {
 	gitjobs, err := w.gitjobs.Cache().List("", labels.Everything())
 	if err != nil {
-		logAndReturn(rw, err)
-		return
+		return err
 	}
 
 	for _, repo := range repoURLs {
 		u, err := url.Parse(repo)
 		if err != nil {
-			logAndReturn(rw, err)
-			return
+			return err
 		}
 		regexpStr := `(?i)(http://|https://|\w+@|ssh://(\w+@)?)` + u.Hostname() + "(:[0-9]+|)[:/]" + u.Path[1:] + "(\\.git)?"
 		repoRegexp, err := regexp.Compile(regexpStr)
 		if err != nil {
-			logAndReturn(rw, err)
-			return
+			return err
 		}
 		for _, gitjob := range gitjobs {
 			if gitjob.Spec.Git.Revision != "" {
@@ -229,22 +272,20 @@ func (w *Webhook) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 				dp.Status.Commit = revision
 				newObj, err := w.gitjobs.UpdateStatus(dp)
 				if err != nil {
-					logAndReturn(rw, err)
-					return
+					return err
 				}
 				// if syncInterval is not set and webhook is configured, set it to 1 hour
 				if newObj.Spec.SyncInterval == 0 {
 					newObj.Spec.SyncInterval = 3600
 					if _, err := w.gitjobs.Update(newObj); err != nil {
-						logAndReturn(rw, err)
-						return
+						return err
 					}
 				}
 			}
 		}
 	}
-	rw.WriteHeader(200)
-	rw.Write([]byte("succeeded"))
+
+	return nil
 }
 
 func logAndReturn(rw http.ResponseWriter, err error) {

pkg/webhook/webhook_test.go

@@ -1,8 +1,19 @@
 package webhook
 
+//go:generate mockgen --build_flags=--mod=mod -destination=../../internal/mocks/gitjob_cache_mock.go -package=mocks github.com/rancher/gitjob/pkg/generated/controllers/gitjob.cattle.io/v1 GitJobCache
+
 import (
+	"bytes"
+	"net/http"
 	"testing"
 
+	"github.com/go-playground/webhooks/v6/azuredevops"
+	"github.com/golang/mock/gomock"
+	"github.com/rancher/gitjob/internal/mocks"
+	v1 "github.com/rancher/gitjob/pkg/apis/gitjob.cattle.io/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+
 	"gotest.tools/assert"
 )
 
@@ -31,3 +42,54 @@ func TestGetBranchTagFromRef(t *testing.T) {
 		assert.Equal(t, tag, outputs[i][1])
 	}
 }
+
+func TestAzureDevopsWebhook(t *testing.T) {
+	const commit = "f00c3a181697bb3829a6462e931c7456bbed557b"
+	const repoURL = "https://dev.azure.com/fleet/git-test/_git/git-test"
+	ctlr := gomock.NewController(t)
+	mockGitjob := mocks.NewMockGitJobController(ctlr)
+	mockGitjobCache := mocks.NewMockGitJobCache(ctlr)
+	gitjob := &v1.GitJob{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test",
+		},
+		Spec: v1.GitJobSpec{
+			Git: v1.GitInfo{
+				Repo:   repoURL,
+				Branch: "main",
+			},
+		},
+	}
+	gitjobs := []*v1.GitJob{gitjob}
+	w := Webhook{gitjobs: mockGitjob}
+	w.azureDevops, _ = azuredevops.New()
+	jsonBody := []byte(`{"subscriptionId":"xxx","notificationId":1,"id":"xxx","eventType":"git.push","publisherId":"tfs","message":{"text":"commit pushed","html":"commit pushed"},"detailedMessage":{"text":"pushed a commit to git-test"},"resource":{"commits":[{"commitId":"` + commit + `","author":{"name":"fleet","email":"[email protected]","date":"2024-01-05T10:16:56Z"},"committer":{"name":"fleet","email":"[email protected]","date":"2024-01-05T10:16:56Z"},"comment":"test commit","url":"https://dev.azure.com/fleet/_apis/git/repositories/xxx/commits/f00c3a181697bb3829a6462e931c7456bbed557b"}],"refUpdates":[{"name":"refs/heads/main","oldObjectId":"135f8a827edae980466f72eef385881bb4e158d8","newObjectId":"` + commit + `"}],"repository":{"id":"xxx","name":"git-test","url":"https://dev.azure.com/fleet/_apis/git/repositories/xxx","project":{"id":"xxx","name":"git-test","url":"https://dev.azure.com/fleet/_apis/projects/xxx","state":"wellFormed","visibility":"unchanged","lastUpdateTime":"0001-01-01T00:00:00"},"defaultBranch":"refs/heads/main","remoteUrl":"` + repoURL + `"},"pushedBy":{"displayName":"Fleet","url":"https://spsprodneu1.vssps.visualstudio.com/xxx/_apis/Identities/xxx","_links":{"avatar":{"href":"https://dev.azure.com/fleet/_apis/GraphProfile/MemberAvatars/msa.xxxx"}},"id":"xxx","uniqueName":"[email protected]","imageUrl":"https://dev.azure.com/fleet/_api/_common/identityImage?id=xxx","descriptor":"xxxx"},"pushId":22,"date":"2024-01-05T10:17:18.735088Z","url":"https://dev.azure.com/fleet/_apis/git/repositories/xxx/pushes/22","_links":{"self":{"href":"https://dev.azure.com/fleet/_apis/git/repositories/xxx/pushes/22"},"repository":{"href":"https://dev.azure.com/fleet/xxx/_apis/git/repositories/xxx"},"commits":{"href":"https://dev.azure.com/fleet/_apis/git/repositories/xxx/pushes/22/commits"},"pusher":{"href":"https://spsprodneu1.vssps.visualstudio.com/xxx/_apis/Identities/xxx"},"refs":{"href":"https://dev.azure.com/fleet/xxx/_apis/git/repositories/xxx/refs/heads/main"}}},"resourceVersion":"1.0","resourceContainers":{"collection":{"id":"xxx","baseUrl":"https://dev.azure.com/fleet/"},"account":{"id":"ec365173-fce3-4dfc-8fc2-950f0b5728b1","baseUrl":"https://dev.azure.com/fleet/"},"project":{"id":"xxx","baseUrl":"https://dev.azure.com/fleet/"}},"createdDate":"2024-01-05T10:17:26.0098694Z"}`)
+	bodyReader := bytes.NewReader(jsonBody)
+	req, err := http.NewRequest(http.MethodPost, repoURL, bodyReader)
+	if err != nil {
+		t.Errorf("unexpected err %v", err)
+	}
+	h := http.Header{}
+	h.Add("X-Vss-Activityid", "xxx")
+	req.Header = h
+
+	expectedStatusUpdate := gitjob.DeepCopy()
+	expectedStatusUpdate.Status.Commit = commit
+	mockGitjobCache.EXPECT().List("", labels.Everything()).Return(gitjobs, nil)
+	mockGitjob.EXPECT().Cache().Return(mockGitjobCache)
+	mockGitjob.EXPECT().UpdateStatus(expectedStatusUpdate).Return(gitjob, nil)
+	mockGitjob.EXPECT().Update(gitjob)
+
+	w.ServeHTTP(&responseWriter{}, req)
+}
+
+type responseWriter struct{}
+
+func (r *responseWriter) Header() http.Header {
+	return http.Header{}
+}
+func (r *responseWriter) Write([]byte) (int, error) {
+	return 0, nil
+}
+
+func (r *responseWriter) WriteHeader(statusCode int) {}