Merge pull request #1267 from tesshuflower/mover-debug-mode

Mover debug mode

Dockerfile

@@ -123,6 +123,8 @@ RUN microdnf --refresh update -y && \
         perl            `# rsync/ssh - rrsync script` \
         stunnel         `# rsync-tls` \
         openssl         `# syncthing - server certs` \
+        vim-minimal     `# for mover debug` \
+        tar             `# for mover debug` \
     && microdnf --setopt=install_weak_deps=0 install -y \
         `# docs are needed so rrsync gets installed for ssh variant` \
         rsync           `# rsync/ssh, rsync-tls - rsync, rrsync` \

api/v1alpha1/common_types.go

@@ -57,6 +57,9 @@ const (
 
 	// Namespace annotation to indicate that elevated permissions are ok for movers
 	PrivilegedMoversNamespaceAnnotation = "volsync.backube/privileged-movers"
+
+	// Annotation on ReplicationSource or ReplicationDestination to enable running the mover job in debug mode
+	EnableDebugMoverAnnotation = "volsync.backube/enable-debug-mover"
 )
 
 const (

controllers/mover/rclone/mover.go

@@ -265,6 +265,9 @@ func (m *Mover) ensureJob(ctx context.Context, dataPVC *corev1.PersistentVolumeC
 		// Cluster-wide proxy settings
 		envVars = utils.AppendEnvVarsForClusterWideProxy(envVars)
 
+		// Run mover in debug mode if required
+		envVars = utils.AppendDebugMoverEnvVar(m.owner, envVars)
+
 		job.Spec.Template.Spec.Containers = []corev1.Container{{
 			Name:    "rclone",
 			Env:     envVars,

controllers/mover/restic/mover.go

@@ -412,6 +412,9 @@ func (m *Mover) ensureJob(ctx context.Context, cachePVC *corev1.PersistentVolume
 		// Cluster-wide proxy settings
 		envVars = utils.AppendEnvVarsForClusterWideProxy(envVars)
 
+		// Run mover in debug mode if required
+		envVars = utils.AppendDebugMoverEnvVar(m.owner, envVars)
+
 		podSpec.Containers = []corev1.Container{{
 			Name:    "restic",
 			Env:     envVars,

controllers/mover/rsync/mover.go

@@ -397,6 +397,10 @@ func (m *Mover) ensureJob(ctx context.Context, dataPVC *corev1.PersistentVolumeC
 			// Set read-only for volume in repl source job spec if the PVC only supports read-only
 			readOnlyVolume = utils.PvcIsReadOnly(dataPVC)
 		}
+
+		// Run mover in debug mode if required
+		containerEnv = utils.AppendDebugMoverEnvVar(m.owner, containerEnv)
+
 		job.Spec.Template.Spec.Containers = []corev1.Container{{
 			Name:    "rsync",
 			Env:     containerEnv,

controllers/mover/rsynctls/mover.go

@@ -481,6 +481,10 @@ func (m *Mover) ensureJob(ctx context.Context, dataPVC *corev1.PersistentVolumeC
 				Value: "0",
 			})
 		}
+
+		// Run mover in debug mode if required
+		podSpec.Containers[0].Env = utils.AppendDebugMoverEnvVar(m.owner, podSpec.Containers[0].Env)
+
 		logger.V(1).Info("Job has PVC", "PVC", dataPVC, "DS", dataPVC.Spec.DataSource)
 		return nil
 	})

controllers/mover/syncthing/mover.go

@@ -58,13 +58,15 @@ const (
 	dataDirMountPath   = "/data"
 	configDirMountPath = "/mover-syncthing/config"
 	certDirMountPath   = "/certs"
+	tempDirMountPath   = "/tmp"
 )
 
 // Volume names loaded by the Deployment.
 const (
 	certVolumeName   = "https-certs"
 	configVolumeName = "syncthing-config"
 	dataVolumeName   = "syncthing-data"
+	tempVolumeName   = "tempdir"
 )
 
 // Ports used by the Syncthing container.
@@ -446,6 +448,9 @@ func (m *Mover) ensureDeployment(ctx context.Context, dataPVC *corev1.Persistent
 		// Cluster-wide proxy settings
 		envVars = utils.AppendEnvVarsForClusterWideProxy(envVars)
 
+		// Run mover in debug mode if required
+		envVars = utils.AppendDebugMoverEnvVar(m.owner, envVars)
+
 		podSpec.Containers = []corev1.Container{
 			{
 				Name:    "syncthing",
@@ -461,6 +466,7 @@ func (m *Mover) ensureDeployment(ctx context.Context, dataPVC *corev1.Persistent
 					{Name: configVolumeName, MountPath: configDirMountPath},
 					{Name: dataVolumeName, MountPath: dataDirMountPath},
 					{Name: certVolumeName, MountPath: certDirMountPath},
+					{Name: tempVolumeName, MountPath: tempDirMountPath},
 				},
 				SecurityContext: &corev1.SecurityContext{
 					AllowPrivilegeEscalation: ptr.To(false),
@@ -505,6 +511,13 @@ func (m *Mover) ensureDeployment(ctx context.Context, dataPVC *corev1.Persistent
 					},
 				},
 			},
+			{
+				Name: tempVolumeName, VolumeSource: corev1.VolumeSource{
+					EmptyDir: &corev1.EmptyDirVolumeSource{
+						Medium: corev1.StorageMediumMemory,
+					},
+				},
+			},
 		}
 
 		defaultMoverResources := corev1.ResourceRequirements{

controllers/mover/syncthing/syncthing_test.go

@@ -1159,6 +1159,8 @@ var _ = Describe("When an RS specifies Syncthing", func() {
 								}
 								Expect(httpsKeysChecked).To(Equal(len(httpsItems)))
 								checked++
+							} else if volume.Name == tempVolumeName {
+								checked++
 							}
 						}
 						// make sure that all volumes are accounted for

controllers/utils/utils.go

@@ -28,6 +28,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/reference"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -209,6 +210,18 @@ func AppendRCloneEnvVars(secret *corev1.Secret, envVars []corev1.EnvVar) []corev
 	return envVars
 }
 
+// Will append the MoverDebug Env var if the volsyncv1alpha1.EnableDebugMoverAnnotation
+// annotation is set on the corresponding ReplicationSource or Destination
+func AppendDebugMoverEnvVar(replicationSourceOrDestObj metav1.Object, envVars []corev1.EnvVar) []corev1.EnvVar {
+	// If the annotation exists on the RS/RD (with any value) then we assume mover debug mode is desired
+	_, debugMoverEnabled := replicationSourceOrDestObj.GetAnnotations()[volsyncv1alpha1.EnableDebugMoverAnnotation]
+	if debugMoverEnabled {
+		envVars = append(envVars, corev1.EnvVar{Name: "DEBUG_MOVER", Value: "1"})
+	}
+
+	return envVars
+}
+
 // Updates to set the securityContext, podLabels on mover pod in the spec and resourceRequirements on the mover
 // containers based on what is set in the MoverConfig
 func UpdatePodTemplateSpecFromMoverConfig(podTemplateSpec *corev1.PodTemplateSpec,

mover-rclone/active.sh

@@ -4,6 +4,46 @@ set -e -o pipefail
 
 echo "VolSync rclone container version: ${version:-unknown}"
 
+SCRIPT_FULLPATH="$(realpath "$0")"
+SCRIPT="$(basename "$SCRIPT_FULLPATH")"
+SCRIPT_DIR="$(dirname "$SCRIPT_FULLPATH")"
+
+# Do not do this debug mover code if this is already the
+# mover script copy in /tmp
+if [[ $DEBUG_MOVER -eq 1 && "$SCRIPT_DIR" != "/tmp" ]]; then
+  MOVER_SCRIPT_COPY="/tmp/$SCRIPT"
+  cp "$SCRIPT_FULLPATH" "$MOVER_SCRIPT_COPY"
+
+  END_DEBUG_FILE="/tmp/exit-debug-if-removed"
+  touch $END_DEBUG_FILE
+
+  echo ""
+  echo "##################################################################"
+  echo "DEBUG_MOVER is enabled, this pod will sleep indefinitely."
+  echo ""
+  echo "The mover script that would normally run has been copied to"
+  echo "$MOVER_SCRIPT_COPY".
+  echo ""
+  echo "To debug, you can modify this file and run it with:"
+  echo "$MOVER_SCRIPT_COPY" "$@"
+  echo ""
+  echo "If you wish to exit this pod after debugging, delete the"
+  echo "file $END_DEBUG_FILE from the system."
+  echo "##################################################################"
+
+  # Wait for user to delete the file before exiting
+  while [[ -f "${END_DEBUG_FILE}" ]]; do
+    sleep 10
+  done
+
+  echo ""
+  echo "##################################################################"
+  echo "Debug done, exiting."
+  echo "##################################################################"
+  sleep 2
+  exit 0
+fi
+
 function error {
     rc="$1"
     shift

mover-restic/entry.sh

@@ -9,6 +9,46 @@ set -e -o pipefail
 echo "VolSync restic container version: ${version:-unknown}"
 echo  "$@"
 
+SCRIPT_FULLPATH="$(realpath "$0")"
+SCRIPT="$(basename "$SCRIPT_FULLPATH")"
+SCRIPT_DIR="$(dirname "$SCRIPT_FULLPATH")"
+
+# Do not do this debug mover code if this is already the
+# mover script copy in /tmp
+if [[ $DEBUG_MOVER -eq 1 && "$SCRIPT_DIR" != "/tmp" ]]; then
+  MOVER_SCRIPT_COPY="/tmp/$SCRIPT"
+  cp "$SCRIPT_FULLPATH" "$MOVER_SCRIPT_COPY"
+
+  END_DEBUG_FILE="/tmp/exit-debug-if-removed"
+  touch $END_DEBUG_FILE
+
+  echo ""
+  echo "##################################################################"
+  echo "DEBUG_MOVER is enabled, this pod will sleep indefinitely."
+  echo ""
+  echo "The mover script that would normally run has been copied to"
+  echo "$MOVER_SCRIPT_COPY".
+  echo ""
+  echo "To debug, you can modify this file and run it with:"
+  echo "$MOVER_SCRIPT_COPY" "$@"
+  echo ""
+  echo "If you wish to exit this pod after debugging, delete the"
+  echo "file $END_DEBUG_FILE from the system."
+  echo "##################################################################"
+
+  # Wait for user to delete the file before exiting
+  while [[ -f "${END_DEBUG_FILE}" ]]; do
+    sleep 10
+  done
+
+  echo ""
+  echo "##################################################################"
+  echo "Debug done, exiting."
+  echo "##################################################################"
+  sleep 2
+  exit 0
+fi
+
 declare -a RESTIC
 RESTIC=("restic")
 if [[ -n "${CUSTOM_CA}" ]]; then

mover-rsync-tls/client.sh

@@ -15,8 +15,46 @@ STUNNEL_LISTEN_PORT=9000
 SOURCE="/data"
 BLOCK_SOURCE="/dev/block"
 
-SCRIPT="$(realpath "$0")"
-SCRIPT_DIR="$(dirname "$SCRIPT")"
+SCRIPT_FULLPATH="$(realpath "$0")"
+SCRIPT="$(basename "$SCRIPT_FULLPATH")"
+SCRIPT_DIR="$(dirname "$SCRIPT_FULLPATH")"
+
+# Do not do this debug mover code if this is already the
+# mover script copy in /tmp
+if [[ $DEBUG_MOVER -eq 1 && "$SCRIPT_DIR" != "/tmp" ]]; then
+  MOVER_SCRIPT_COPY="/tmp/$SCRIPT"
+  cp "$SCRIPT_FULLPATH" "$MOVER_SCRIPT_COPY"
+
+  END_DEBUG_FILE="/tmp/exit-debug-if-removed"
+  touch $END_DEBUG_FILE
+
+  echo ""
+  echo "##################################################################"
+  echo "DEBUG_MOVER is enabled, this pod will sleep indefinitely."
+  echo ""
+  echo "The mover script that would normally run has been copied to"
+  echo "$MOVER_SCRIPT_COPY".
+  echo ""
+  echo "To debug, you can modify this file and run it with:"
+  echo "$MOVER_SCRIPT_COPY" "$@"
+  echo ""
+  echo "If you wish to exit this pod after debugging, delete the"
+  echo "file $END_DEBUG_FILE from the system."
+  echo "##################################################################"
+
+  # Wait for user to delete the file before exiting
+  while [[ -f "${END_DEBUG_FILE}" ]]; do
+    sleep 10
+  done
+
+  echo ""
+  echo "##################################################################"
+  echo "Debug done, exiting."
+  echo "##################################################################"
+  sleep 2
+  exit 0
+fi
+
 cd "$SCRIPT_DIR"
 
 # shellcheck disable=SC2317  # It's reachable due to the TRAP
@@ -144,7 +182,7 @@ else
     if [[ $rc -eq 0 ]]; then
         # Tell server to shutdown. Actual file contents don't matter
         echo "Sending shutdown to remote..."
-        rsync "$SCRIPT" rsync://127.0.0.1:$STUNNEL_LISTEN_PORT/control/complete
+        rsync "$SCRIPT_FULLPATH" rsync://127.0.0.1:$STUNNEL_LISTEN_PORT/control/complete
         echo "...done"
         sleep 5  # Give time for the remote to shut down
     else

mover-rsync-tls/server.sh

@@ -11,7 +11,46 @@ PSK_FILE=/keys/psk.txt
 RSYNC_LOG=/tmp/rsyncd.log
 IPV6_DISABLED=$(cat /sys/module/ipv6/parameters/disable)
 
-SCRIPT_DIR="$(dirname "$(realpath "$0")")"
+SCRIPT_FULLPATH="$(realpath "$0")"
+SCRIPT="$(basename "$SCRIPT_FULLPATH")"
+SCRIPT_DIR="$(dirname "$SCRIPT_FULLPATH")"
+
+# Do not do this debug mover code if this is already the
+# mover script copy in /tmp
+if [[ $DEBUG_MOVER -eq 1 && "$SCRIPT_DIR" != "/tmp" ]]; then
+  MOVER_SCRIPT_COPY="/tmp/$SCRIPT"
+  cp "$SCRIPT_FULLPATH" "$MOVER_SCRIPT_COPY"
+
+  END_DEBUG_FILE="/tmp/exit-debug-if-removed"
+  touch $END_DEBUG_FILE
+
+  echo ""
+  echo "##################################################################"
+  echo "DEBUG_MOVER is enabled, this pod will sleep indefinitely."
+  echo ""
+  echo "The mover script that would normally run has been copied to"
+  echo "$MOVER_SCRIPT_COPY".
+  echo ""
+  echo "To debug, you can modify this file and run it with:"
+  echo "$MOVER_SCRIPT_COPY" "$@"
+  echo ""
+  echo "If you wish to exit this pod after debugging, delete the"
+  echo "file $END_DEBUG_FILE from the system."
+  echo "##################################################################"
+
+  # Wait for user to delete the file before exiting
+  while [[ -f "${END_DEBUG_FILE}" ]]; do
+    sleep 10
+  done
+
+  echo ""
+  echo "##################################################################"
+  echo "Debug done, exiting."
+  echo "##################################################################"
+  sleep 2
+  exit 0
+fi
+
 cd "$SCRIPT_DIR"
 
 STUNNEL_LISTEN_PORT=:::8000

mover-rsync/destination.sh

@@ -4,6 +4,46 @@ set -e -o pipefail
 
 echo "VolSync rsync container version: ${version:-unknown}"
 
+SCRIPT_FULLPATH="$(realpath "$0")"
+SCRIPT="$(basename "$SCRIPT_FULLPATH")"
+SCRIPT_DIR="$(dirname "$SCRIPT_FULLPATH")"
+
+# Do not do this debug mover code if this is already the
+# mover script copy in /tmp
+if [[ $DEBUG_MOVER -eq 1 && "$SCRIPT_DIR" != "/tmp" ]]; then
+  MOVER_SCRIPT_COPY="/tmp/$SCRIPT"
+  cp "$SCRIPT_FULLPATH" "$MOVER_SCRIPT_COPY"
+
+  END_DEBUG_FILE="/tmp/exit-debug-if-removed"
+  touch $END_DEBUG_FILE
+
+  echo ""
+  echo "##################################################################"
+  echo "DEBUG_MOVER is enabled, this pod will sleep indefinitely."
+  echo ""
+  echo "The mover script that would normally run has been copied to"
+  echo "$MOVER_SCRIPT_COPY".
+  echo ""
+  echo "To debug, you can modify this file and run it with:"
+  echo "$MOVER_SCRIPT_COPY" "$@"
+  echo ""
+  echo "If you wish to exit this pod after debugging, delete the"
+  echo "file $END_DEBUG_FILE from the system."
+  echo "##################################################################"
+
+  # Wait for user to delete the file before exiting
+  while [[ -f "${END_DEBUG_FILE}" ]]; do
+    sleep 10
+  done
+
+  echo ""
+  echo "##################################################################"
+  echo "Debug done, exiting."
+  echo "##################################################################"
+  sleep 2
+  exit 0
+fi
+
 # Allow source's key to access, but restrict what it can do.
 mkdir -p ~/.ssh
 chmod 700 ~/.ssh