-
Notifications
You must be signed in to change notification settings - Fork 90
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Support Running Orchestrator behind a Reverse Proxy (#4724)
If we need to put an Orchestrator behind a TLS terminating reverse proxy,the NATS server should be configurred in a very specific way, and the NATS clients (compute nodes) should also be configured in a certain way. The NATS server should say the TLS is available, although it is not. Also, the compute node should enforce TLS communication for NATS, because reverse proxy supports TLS. See link: https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls#tls-terminating-reverse-proxies Sample Orchestrator Node config: ```yaml NameProvider: "uuid" API: Port: 1234 Orchestrator: Enabled: true Auth: Token: "i_am_very_secret_token" SupportReverseProxy: true ``` Sample Compute Node Config: ```yaml NameProvider: "uuid" API: Port: 1234 Compute: Enabled: true Orchestrators: - nats://bacalhau-traefik-node:4222 Auth: Token: "i_am_very_secret_token" TLS: RequireTLS: true ``` Please see the integration tests in this commit, it has a very detailed test suite covering all cases. Linear: https://linear.app/expanso/issue/ENG-379/bacalhau-to-support-tls-behind-reverse-proxy <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit ## Release Notes - **New Features** - Introduced support for TLS communication and reverse proxy configurations in compute and orchestrator nodes. - Added new properties in the API schema to enhance configuration options. - **Bug Fixes** - Improved error handling for NATS connections based on TLS requirements. - **Documentation** - Updated Swagger API documentation to include new properties and configurations. - **Tests** - Added a new test suite to validate orchestrator functionality behind a reverse proxy. - **Chores** - Introduced new Docker Compose configurations for enhanced service orchestration. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
- Loading branch information
Showing
21 changed files
with
488 additions
and
24 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -438,4 +438,5 @@ buildvcs | |
| Nilf | ||
| IMDS | ||
| tlsca | ||
| Lenf | ||
| Lenf | ||
| traefik | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
124 changes: 124 additions & 0 deletions
124
test_integration/9_orchestrator_behind_reverse_proxy_test.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,124 @@ | ||
| package test_integration | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "strings" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "bacalhau/integration_tests/utils" | ||
|
|
||
| "github.com/google/uuid" | ||
| "github.com/stretchr/testify/suite" | ||
| ) | ||
|
|
||
| type OrchestratorBehindReverseProxySuite struct { | ||
| BaseDockerComposeTestSuite | ||
| } | ||
|
|
||
| func NewOrchestratorBehindReverseProxySuite() *OrchestratorBehindReverseProxySuite { | ||
| s := &OrchestratorBehindReverseProxySuite{} | ||
| s.GlobalRunIdentifier = globalTestExecutionId | ||
| s.SuiteRunIdentifier = strings.ToLower(strings.Split(uuid.New().String(), "-")[0]) | ||
| return s | ||
| } | ||
|
|
||
| func (s *OrchestratorBehindReverseProxySuite) SetupSuite() { | ||
| // In this test suite, the orchestrator is running behind a reverse proxy, and all | ||
| // the NATS traffic between orchestrator and compute Node go through a real reverse proxy (Traefik) | ||
|
|
||
| rawDockerComposeFilePath := "./common_assets/docker_compose_files/orchestrator-compute-traefik-custom-startup.yml" | ||
| s.Context, s.Cancel = context.WithCancel(context.Background()) | ||
|
|
||
| traefikConfigFile := s.commonAssets("nodes_configs/9_traefik_static_config.yaml") | ||
| traefikStartCommand := fmt.Sprintf("--configFile=%s", traefikConfigFile) | ||
|
|
||
| orchestratorConfigFile := s.commonAssets("nodes_configs/9_orchestrator_node_behind_reverse_proxy.yaml") | ||
| orchestratorStartCommand := fmt.Sprintf("bacalhau serve --config=%s", orchestratorConfigFile) | ||
|
|
||
| computeConfigFile := s.commonAssets("nodes_configs/9_compute_node_with_enforced_tls_nats.yaml") | ||
| computeStartCommand := fmt.Sprintf("bacalhau serve --config=%s", computeConfigFile) | ||
| extraRenderingData := map[string]interface{}{ | ||
| "OrchestratorStartCommand": orchestratorStartCommand, | ||
| "ComputeStartCommand": computeStartCommand, | ||
| "TraefikStartCommand": traefikStartCommand, | ||
| } | ||
| s.BaseDockerComposeTestSuite.SetupSuite(rawDockerComposeFilePath, extraRenderingData) | ||
| } | ||
|
|
||
| func (s *OrchestratorBehindReverseProxySuite) TearDownSuite() { | ||
| s.T().Log("Tearing down [Test Suite] in OrchestratorBehindReverseProxySuite...") | ||
| s.BaseDockerComposeTestSuite.TearDownSuite() | ||
| } | ||
|
|
||
| func (s *OrchestratorBehindReverseProxySuite) TestRunHelloWorldJobWithOrchestratorBehindReverseProxy() { | ||
| result, err := s.executeCommandInDefaultJumpbox( | ||
| []string{ | ||
| "bacalhau", | ||
| "job", | ||
| "run", | ||
| "--wait=false", | ||
| "--id-only", | ||
| "/bacalhau_integration_tests/common_assets/job_specs/hello_world.yml", | ||
| }) | ||
| s.Require().NoError(err) | ||
|
|
||
| jobID, err := utils.ExtractJobIDFromShortOutput(result) | ||
| s.Require().NoError(err) | ||
|
|
||
| _, err = s.waitForJobToComplete(jobID, 30*time.Second) | ||
| s.Require().NoError(err) | ||
|
|
||
| resultDescription, err := s.executeCommandInDefaultJumpbox([]string{"bacalhau", "job", "describe", jobID}) | ||
| s.Require().NoError(err) | ||
| s.Require().Contains(resultDescription, "hello bacalhau world", resultDescription) | ||
| } | ||
|
|
||
| func (s *OrchestratorBehindReverseProxySuite) TestNatsConnectionWillFailWithoutRequireTLS() { | ||
| _, err := s.executeCommandInDefaultJumpbox( | ||
| []string{ | ||
| "nats", | ||
| "--server=nats://i_am_very_secret_token@bacalhau-traefik-node:4222", | ||
| "--no-tlsfirst", | ||
| "pub", | ||
| "node.info", | ||
| "helloworld", | ||
| }) | ||
| s.Require().Error(err) | ||
| s.Require().ErrorContains(err, "error: read tcp") | ||
| s.Require().ErrorContains(err, "timeout") | ||
| } | ||
|
|
||
| func (s *OrchestratorBehindReverseProxySuite) TestNatsTLSConnectionWillFailWithoutGoingThroughReverseProxy() { | ||
| _, err := s.executeCommandInDefaultJumpbox( | ||
| []string{ | ||
| "nats", | ||
| "--server=nats://i_am_very_secret_token@bacalhau-orchestrator-node:4222", | ||
| "--tlsca=/bacalhau_integration_tests/common_assets/certificates/nats_custom/nats_root_ca.crt", | ||
| "--tlsfirst", | ||
| "pub", | ||
| "node.info", | ||
| "helloworld", | ||
| }) | ||
| s.Require().Error(err) | ||
| s.Require().ErrorContains(err, "error: tls: first record does not look like a TLS handshake") | ||
| } | ||
|
|
||
| func (s *OrchestratorBehindReverseProxySuite) TestNatsConnectionWillSucceedWithRequireTLS() { | ||
| result, err := s.executeCommandInDefaultJumpbox( | ||
| []string{ | ||
| "nats", | ||
| "--server=nats://i_am_very_secret_token@bacalhau-traefik-node:4222", | ||
| "--tlsfirst", | ||
| "pub", | ||
| "node.info", | ||
| "helloworld", | ||
| }) | ||
| s.Require().NoError(err) | ||
| s.Require().Contains(result, `Published 10 bytes to "node.info"`) | ||
| } | ||
|
|
||
| func TestOrchestratorBehindReverseProxySuite(t *testing.T) { | ||
| suite.Run(t, NewOrchestratorBehindReverseProxySuite()) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.