-
Notifications
You must be signed in to change notification settings - Fork 208
215 lines (200 loc) · 10.3 KB
/
ci-self-managed.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
name: CI Self-Managed
on:
# push:
# branches:
# - master
# schedule:
# - cron: '0 0 * * 0' # every week on sunday at 0 am
workflow_dispatch:
inputs:
test_name:
description: 'test(s) to run (example connect/connect-jms-weblogic-sink connect/connect-http-sink)'
required: false
default: ''
jobs:
start-runner:
name: Start Github self-hosted runner
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
repository: vdesabou/kafka-docker-playground
fetch-depth: 0
- name: Decrypt secrets.tar
run: |
./.github/scripts/decrypt_secret.sh
tar xvf secrets.tar
rm secrets.tar
mkdir -p $HOME/.aws
mv aws_credentials $HOME/.aws/credentials
chmod -R a+rw $HOME/.aws
mkdir -p $HOME/.confluent
echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
env:
SECRETS_ENCRYPTION_PASSWORD: ${{ secrets.SECRETS_ENCRYPTION_PASSWORD }}
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME}}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD}}
- name: Start EC2 instance github-actions-runner-vsaboulin
run: |
aws ec2 start-instances --instance-ids i-089ef31a75cb3f0e6
build:
runs-on: self-hosted
needs: start-runner # required to start the main job when the runner is read
name: ${{ matrix.tag }} ${{ matrix.test_list }}
strategy:
fail-fast: false
matrix:
tag: ["7.6.1"]
test_list : [
"🚀 connect/connect-mapr-sink",
]
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
repository: vdesabou/kafka-docker-playground
fetch-depth: 0
- name: Decrypt secrets.tar
run: |
./.github/scripts/decrypt_secret.sh
tar xvf secrets.tar
rm secrets.tar
mkdir -p $HOME/.aws
mv aws_credentials_with_assuming_iam_role $HOME/.aws/credentials-with-assuming-iam-role
chmod -R a+rw $HOME/.aws
mkdir -p $HOME/.confluent
echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
env:
SECRETS_ENCRYPTION_PASSWORD: ${{ secrets.SECRETS_ENCRYPTION_PASSWORD }}
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME}}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD}}
- name: "Install confluent CLI"
run: |
curl -L --http1.1 https://cnfl.io/cli | sudo sh -s -- -b /usr/local/bin
export PATH=$PATH:/usr/local/bin
- name: Build and Test
run: |
export PATH=$PATH:$GITHUB_WORKSPACE/scripts/cli
bash scripts/run-tests.sh "${{ matrix.test_list }}" "${{ matrix.tag }}"
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID}}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
S3_PROVIDER_INTEGRATION_ID: ${{ secrets.S3_PROVIDER_INTEGRATION_ID}}
AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_ACCESS_KEY_ID}}
AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_SECRET_ACCESS_KEY}}
AWS_STS_ROLE_ARN: ${{ secrets.AWS_STS_ROLE_ARN}}
AZ_USER: ${{ secrets.AZ_USER}}
AZ_PASS: ${{ secrets.AZ_PASS}}
AZURE_SUBSCRIPTION_NAME: ${{ secrets.AZURE_SUBSCRIPTION_NAME}}
CONFLUENT_CLOUD_EMAIL: ${{ secrets.CONFLUENT_CLOUD_EMAIL}}
CONFLUENT_CLOUD_PASSWORD: ${{ secrets.CONFLUENT_CLOUD_PASSWORD}}
ENVIRONMENT: ${{ secrets.ENVIRONMENT}}
CLUSTER_NAME: ${{ secrets.CLUSTER_NAME}}
CLUSTER_REGION: ${{ secrets.CLUSTER_REGION}}
CLUSTER_CLOUD: ${{ secrets.CLUSTER_CLOUD}}
CLUSTER_CREDS: ${{ secrets.CLUSTER_CREDS}}
AWS_DATABRICKS_CLUSTER_NAME: ${{ secrets.AWS_DATABRICKS_CLUSTER_NAME}}
AWS_DATABRICKS_CLUSTER_REGION: ${{ secrets.AWS_DATABRICKS_CLUSTER_REGION}}
AWS_DATABRICKS_CLUSTER_CLOUD: ${{ secrets.AWS_DATABRICKS_CLUSTER_CLOUD}}
AWS_DATABRICKS_CLUSTER_CREDS: ${{ secrets.AWS_DATABRICKS_CLUSTER_CREDS}}
GCP_CLUSTER_NAME: ${{ secrets.GCP_CLUSTER_NAME}}
GCP_CLUSTER_REGION: ${{ secrets.GCP_CLUSTER_REGION}}
GCP_CLUSTER_CLOUD: ${{ secrets.GCP_CLUSTER_CLOUD}}
GCP_CLUSTER_CREDS: ${{ secrets.GCP_CLUSTER_CREDS}}
AZURE_CLUSTER_NAME: ${{ secrets.AZURE_CLUSTER_NAME}}
AZURE_CLUSTER_REGION: ${{ secrets.AZURE_CLUSTER_REGION}}
AZURE_CLUSTER_CLOUD: ${{ secrets.AZURE_CLUSTER_CLOUD}}
AZURE_CLUSTER_CREDS: ${{ secrets.AZURE_CLUSTER_CREDS}}
SCHEMA_REGISTRY_CREDS: ${{ secrets.SCHEMA_REGISTRY_CREDS}}
CONFLUENT_LICENSE: ${{ secrets.CONFLUENT_LICENSE}}
SALESFORCE_USERNAME: ${{ secrets.SALESFORCE_USERNAME}}
SALESFORCE_PASSWORD: ${{ secrets.SALESFORCE_PASSWORD}}
SALESFORCE_CONSUMER_KEY: ${{ secrets.SALESFORCE_CONSUMER_KEY}}
SALESFORCE_CONSUMER_PASSWORD: ${{ secrets.SALESFORCE_CONSUMER_PASSWORD}}
SALESFORCE_SECURITY_TOKEN: ${{ secrets.SALESFORCE_SECURITY_TOKEN}}
SALESFORCE_INSTANCE: ${{ secrets.SALESFORCE_INSTANCE}}
SALESFORCE_USERNAME_ACCOUNT2: ${{ secrets.SALESFORCE_USERNAME_ACCOUNT2}}
SALESFORCE_PASSWORD_ACCOUNT2: ${{ secrets.SALESFORCE_PASSWORD_ACCOUNT2}}
SALESFORCE_SECURITY_TOKEN_ACCOUNT2: ${{ secrets.SALESFORCE_SECURITY_TOKEN_ACCOUNT2}}
SALESFORCE_CONSUMER_KEY_ACCOUNT2: ${{ secrets.SALESFORCE_CONSUMER_KEY_ACCOUNT2}}
SALESFORCE_CONSUMER_PASSWORD_ACCOUNT2: ${{ secrets.SALESFORCE_CONSUMER_PASSWORD_ACCOUNT2}}
SALESFORCE_CONSUMER_KEY_WITH_JWT: ${{ secrets.SALESFORCE_CONSUMER_KEY_WITH_JWT}}
SALESFORCE_CONSUMER_PASSWORD_WITH_JWT: ${{ secrets.SALESFORCE_CONSUMER_PASSWORD_WITH_JWT}}
DD_API_KEY: ${{ secrets.DD_API_KEY}}
DD_APP_KEY: ${{ secrets.DD_APP_KEY}}
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME}}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD}}
JIRA_URL: ${{ secrets.JIRA_URL}}
JIRA_USERNAME: ${{ secrets.JIRA_USERNAME}}
JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN}}
MARKETO_ENDPOINT_URL: ${{ secrets.MARKETO_ENDPOINT_URL}}
MARKETO_CLIENT_ID: ${{ secrets.MARKETO_CLIENT_ID}}
MARKETO_CLIENT_SECRET: ${{ secrets.MARKETO_CLIENT_SECRET}}
PAGERDUTY_USER_EMAIL: ${{ secrets.PAGERDUTY_USER_EMAIL}}
PAGERDUTY_API_KEY: ${{ secrets.PAGERDUTY_API_KEY}}
PAGERDUTY_SERVICE_ID: ${{ secrets.PAGERDUTY_SERVICE_ID}}
CCLOUD_REST_PROXY_SECURITY_PLUGIN_API_KEY: ${{ secrets.CCLOUD_REST_PROXY_SECURITY_PLUGIN_API_KEY}}
CCLOUD_REST_PROXY_SECURITY_PLUGIN_API_SECRET: ${{ secrets.CCLOUD_REST_PROXY_SECURITY_PLUGIN_API_SECRET}}
SERVICENOW_URL: ${{ secrets.SERVICENOW_URL}}
SERVICENOW_PASSWORD: ${{ secrets.SERVICENOW_PASSWORD}}
SERVICENOW_DEVELOPER_USERNAME: ${{ secrets.SERVICENOW_DEVELOPER_USERNAME}}
SERVICENOW_DEVELOPER_PASSWORD: ${{ secrets.SERVICENOW_DEVELOPER_PASSWORD}}
SNOWFLAKE_ACCOUNT_NAME: ${{ secrets.SNOWFLAKE_ACCOUNT_NAME}}
SNOWFLAKE_USERNAME: ${{ secrets.SNOWFLAKE_USERNAME}}
SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD}}
ZENDESK_URL: ${{ secrets.ZENDESK_URL}}
ZENDESK_USERNAME: ${{ secrets.ZENDESK_USERNAME}}
ZENDESK_PASSWORD: ${{ secrets.ZENDESK_PASSWORD}}
CONNECTOR_GITHUB_ACCESS_TOKEN: ${{ secrets.CONNECTOR_GITHUB_ACCESS_TOKEN}}
CI_GITHUB_TOKEN: ${{ secrets.CI_GITHUB_TOKEN}}
AUDIT_LOG_CLUSTER_BOOTSTRAP_SERVERS: ${{ secrets.AUDIT_LOG_CLUSTER_BOOTSTRAP_SERVERS}}
AUDIT_LOG_CLUSTER_API_KEY: ${{ secrets.AUDIT_LOG_CLUSTER_API_KEY}}
AUDIT_LOG_CLUSTER_API_SECRET: ${{ secrets.AUDIT_LOG_CLUSTER_API_SECRET}}
NGROK_AUTH_TOKEN: ${{ secrets.NGROK_CI_AUTH_TOKEN}}
DATABRICKS_AWS_BUCKET_NAME: ${{ secrets.DATABRICKS_AWS_BUCKET_NAME}}
DATABRICKS_AWS_BUCKET_REGION: ${{ secrets.DATABRICKS_AWS_BUCKET_REGION}}
DATABRICKS_AWS_STAGING_S3_ACCESS_KEY_ID: ${{ secrets.DATABRICKS_AWS_STAGING_S3_ACCESS_KEY_ID}}
DATABRICKS_AWS_STAGING_S3_SECRET_ACCESS_KEY: ${{ secrets.DATABRICKS_AWS_STAGING_S3_SECRET_ACCESS_KEY}}
DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_SERVER_HOSTNAME}}
DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH}}
DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN}}
ORACLE_CONTAINER_REGISTRY_USERNAME: ${{ secrets.ORACLE_CONTAINER_REGISTRY_USERNAME}}
ORACLE_CONTAINER_REGISTRY_PASSWORD: ${{ secrets.ORACLE_CONTAINER_REGISTRY_PASSWORD}}
GCP_KEYFILE_CONTENT: ${{ secrets.GCP_KEYFILE_CONTENT}}
GCP_PROJECT: ${{ secrets.GCP_PROJECT}}
HPE_MAPR_EMAIL: ${{ secrets.HPE_MAPR_EMAIL}}
HPE_MAPR_TOKEN: ${{ secrets.HPE_MAPR_TOKEN}}
MONGODB_ATLAS_HOST: ${{ secrets.MONGODB_ATLAS_HOST}}
MONGODB_ATLAS_USER: ${{ secrets.MONGODB_ATLAS_USER}}
MONGODB_ATLAS_PASSWORD: ${{ secrets.MONGODB_ATLAS_PASSWORD}}
stop-runner:
name: Stop Github self-hosted runner
needs:
- start-runner # required to get output from the start-runner job
- build # required to wait when the main job is done
if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
repository: vdesabou/kafka-docker-playground
fetch-depth: 0
- name: Decrypt secrets.tar
run: |
./.github/scripts/decrypt_secret.sh
tar xvf secrets.tar
rm secrets.tar
mkdir -p $HOME/.aws
mv aws_credentials_with_assuming_iam_role $HOME/.aws/credentials-with-assuming-iam-role
chmod -R a+rw $HOME/.aws
mkdir -p $HOME/.confluent
echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
env:
SECRETS_ENCRYPTION_PASSWORD: ${{ secrets.SECRETS_ENCRYPTION_PASSWORD }}
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME}}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD}}
- name: Stop EC2 instance github-actions-runner-vsaboulin
run: |
aws ec2 stop-instances --instance-ids i-089ef31a75cb3f0e6